In [5]:
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.svm import OneClassSVM
from sklearn.linear_model import LinearRegression
from sklearn.impute import KNNImputer
from sklearn.experimental import enable_iterative_imputer
from sklearn.ensemble import RandomForestRegressor
In [7]:
df = pd.read_excel('SmurfitWestrockConfidential_PredictionData_ForColumbiaU.xlsx')
df
Out[7]:
| TimeStamp | QualityMeasure | 32PRODGRADE | PM2GrossTPH | 18LI0028:18LA0028.PNT | 19FC0002:19FC0002.MEAS | 19LI0472:19LI0472.PNT | 19LI0477:19LA0477.PNT | 22AUTOMAX_RD:1STDRYER_DFB.PNT | 22AUTOMAX_RD:PERCENT_DRAW.RO01 | ... | 32DRVMST_RD4:5THDRYER_CUR.MEAS | 32DRVMST_RD4:5THDRYER_DRW.MEAS | 32DRVMST_RD4:6THDRYER_CUR.MEAS | 32DRVMST_RD4:6THDRYER_DRW.MEAS | 32DRVMST_RD5:7THDRYER_CUR.MEAS | 32DRVMST_RD5:7THDRYER_DRW.MEAS | 32DRVMST_RD5:CAL_DRW.MEAS | 32DRVMST_RD6:REELDRUM_DRW.MEAS | 32DRVMST_RD6:REELDRUM_CUR.MEAS | 12FI0110:12FI0110.PNT | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2022-07-01 00:30:06 | 51.500000 | ME23 | 61.333 | 69.30485 | 0.000701 | 88.91846 | 39.33166 | 60.34387 | 1.857722 | ... | 30.911760 | 3.038673 | 22.560300 | 3.077353 | 59.680890 | 3.521016 | 3.712504 | NaN | 32.779830 | 5478.449962 |
| 1 | 2022-07-01 02:27:09 | 51.500000 | ME23 | 61.22888 | 67.75152 | 0.000701 | 87.77137 | 39.16823 | 59.70235 | 1.850324 | ... | 30.000000 | 2.099999 | 20.231380 | 3.370393 | 59.380340 | 3.133648 | 3.2 | NaN | 32.535390 | 5383.978364 |
| 2 | 2022-07-01 03:35:05 | 51.666668 | ME23 | 60.02072 | 58.73729 | 0.000701 | 88.66229 | 39.07338 | 58.69795 | 1.826701 | ... | 31.000000 | 1.951585 | 20.000000 | 3.131916 | 57.001580 | 3.874452 | 3.577793 | NaN | 32.499210 | 5445.179291 |
| 3 | 2022-07-02 04:26:49 | 50.916668 | ME23 | 59.77631 | 70.72887 | 0.000701 | 84.09638 | 37.36133 | 54.17207 | 1.647153 | ... | 30.000000 | 3.000644 | 20.716260 | 2.618661 | 62.859860 | 3.227230 | 3.2 | NaN | 32.513280 | 5740.561517 |
| 4 | 2022-07-02 05:01:01 | 51.333332 | ME23 | 59.82626 | 66.93857 | 0.000701 | 85.3093 | 37.31067 | 54.43218 | 1.657713 | ... | 29.734050 | 3.412886 | 19.632880 | 2.900000 | 59.467300 | 3.383340 | 3.483357 | NaN | 31.844340 | 5496.862867 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 3264 | 2024-04-20 23:12:08 | 54.500000 | ME23 | 60.143318 | 61.280479 | 0.089437 | 87.388474 | 54.804222 | 66.018845 | 2.101269 | ... | 21.454918 | 6.653681 | 21.315865 | 0.096677 | 33.701187 | 2.113836 | 15.06897 | -14.21159 | 39.717907 | 3938.786707 |
| 3265 | 2024-04-21 00:20:35 | 54.000000 | ME23 | 59.95488 | 50.14521 | 0.03816 | 91.183426 | 54.751965 | 66.292595 | 2.109345 | ... | 21.676327 | 3.387486 | 21.179834 | 0.676058 | 34.193932 | 2.696882 | 13.289554 | -15.003655 | 39.018269 | 3856.16395 |
| 3266 | 2024-04-21 01:31:49 | 53.333332 | ME23 | 56.970501 | 52.800426 | 0.101979 | 86.527016 | 54.694817 | 57.609299 | 1.907183 | ... | 20.083513 | 4.565936 | 21.827986 | 1.656811 | 30.285254 | 1.145926 | 13.768075 | -15.192116 | 39.079048 | 3759.583816 |
| 3267 | 2024-04-23 08:44:20 | 55.166668 | ME23 | 42.047024 | 59.331093 | 48.501858 | 87.588478 | 50.500904 | 60.305599 | 1.962264 | ... | 17.503027 | 3.006409 | 18.554209 | 1.585764 | 24.350523 | 0.557838 | 2.824477 | -4 | 44.418255 | 3733.62799 |
| 3268 | 2024-04-23 09:20:41 | 55.166668 | ME23 | 42.062473 | 68.735703 | 49.230766 | 90.254387 | 50.409645 | 60.232826 | 1.962190 | ... | 17.305225 | 3.123813 | 18.688637 | 1.159925 | 23.925316 | 0.903451 | 2.641837 | -4 | 44.630287 | 3825.232948 |
3269 rows × 522 columns
In [9]:
# Use the replace() method to replace values in df_clean
df_clean = df.copy()
df_clean['32PRODGRADE'] = df_clean['32PRODGRADE'].replace({
'ME23AG': 'ME23', # Change 'ME23AG' to 'ME23'
'Bad': 'Exception', # Change 'Bad' to 'Exception'
'I/O Timeout': 'Exception', # Change 'I/O Timeout' to 'Exception'
'KPBS63': 'Exception' # Change 'KPBS63' to 'Exception'
})
df_clean.head()
Out[9]:
| TimeStamp | QualityMeasure | 32PRODGRADE | PM2GrossTPH | 18LI0028:18LA0028.PNT | 19FC0002:19FC0002.MEAS | 19LI0472:19LI0472.PNT | 19LI0477:19LA0477.PNT | 22AUTOMAX_RD:1STDRYER_DFB.PNT | 22AUTOMAX_RD:PERCENT_DRAW.RO01 | ... | 32DRVMST_RD4:5THDRYER_CUR.MEAS | 32DRVMST_RD4:5THDRYER_DRW.MEAS | 32DRVMST_RD4:6THDRYER_CUR.MEAS | 32DRVMST_RD4:6THDRYER_DRW.MEAS | 32DRVMST_RD5:7THDRYER_CUR.MEAS | 32DRVMST_RD5:7THDRYER_DRW.MEAS | 32DRVMST_RD5:CAL_DRW.MEAS | 32DRVMST_RD6:REELDRUM_DRW.MEAS | 32DRVMST_RD6:REELDRUM_CUR.MEAS | 12FI0110:12FI0110.PNT | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2022-07-01 00:30:06 | 51.500000 | ME23 | 61.333 | 69.30485 | 0.000701 | 88.91846 | 39.33166 | 60.34387 | 1.857722 | ... | 30.91176 | 3.038673 | 22.56030 | 3.077353 | 59.68089 | 3.521016 | 3.712504 | NaN | 32.77983 | 5478.449962 |
| 1 | 2022-07-01 02:27:09 | 51.500000 | ME23 | 61.22888 | 67.75152 | 0.000701 | 87.77137 | 39.16823 | 59.70235 | 1.850324 | ... | 30.00000 | 2.099999 | 20.23138 | 3.370393 | 59.38034 | 3.133648 | 3.2 | NaN | 32.53539 | 5383.978364 |
| 2 | 2022-07-01 03:35:05 | 51.666668 | ME23 | 60.02072 | 58.73729 | 0.000701 | 88.66229 | 39.07338 | 58.69795 | 1.826701 | ... | 31.00000 | 1.951585 | 20.00000 | 3.131916 | 57.00158 | 3.874452 | 3.577793 | NaN | 32.49921 | 5445.179291 |
| 3 | 2022-07-02 04:26:49 | 50.916668 | ME23 | 59.77631 | 70.72887 | 0.000701 | 84.09638 | 37.36133 | 54.17207 | 1.647153 | ... | 30.00000 | 3.000644 | 20.71626 | 2.618661 | 62.85986 | 3.227230 | 3.2 | NaN | 32.51328 | 5740.561517 |
| 4 | 2022-07-02 05:01:01 | 51.333332 | ME23 | 59.82626 | 66.93857 | 0.000701 | 85.3093 | 37.31067 | 54.43218 | 1.657713 | ... | 29.73405 | 3.412886 | 19.63288 | 2.900000 | 59.46730 | 3.383340 | 3.483357 | NaN | 31.84434 | 5496.862867 |
5 rows × 522 columns
In [11]:
# Find columns after the 4th that contain text data
non_numeric_columns_after_4th = df_clean.iloc[:, 4:].select_dtypes(include=['object']).columns
# Find the text content in each non-numeric column (excluding numbers) and count occurrences of each unique value
unique_text_summary = {}
for col in non_numeric_columns_after_4th:
text_values = [val for val in df_clean[col].unique() if isinstance(val, str) and not val.isdigit()]
unique_text_summary[col] = {val: df_clean[col].value_counts().get(val, 0) for val in text_values}
# Display the unique text content in each non-numeric column and their occurrence count
for col, values in unique_text_summary.items():
print(f"Column: {col}")
for val, count in values.items():
print(f" {val}: {count}")
print("\n")
for col in non_numeric_columns_after_4th:
text_values = [val for val in df_clean[col].unique() if isinstance(val, str) and not val.isdigit()]
# Replace the found text content with NA
df_clean[col] = df_clean[col].replace(text_values, 'Exception')
# Display the first few rows of the replaced data
df_clean[non_numeric_columns_after_4th].head()
Column: 18LI0028:18LA0028.PNT Bad: 4 I/O Timeout: 1 Column: 19FC0002:19FC0002.MEAS Bad: 78 Column: 19LI0472:19LI0472.PNT Bad: 3 Column: 19LI0477:19LA0477.PNT Bad: 201 Column: 22AUTOMAX_RD:1STDRYER_DFB.PNT Bad: 36 Out of Serv: 1 Column: 22AUTOMAX_RD:REELDRUM_SFB.PNT Bad: 34 Out of Serv: 1 Column: 27FI0023:27FC0023.MEAS Bad: 154 I/O Timeout: 1 Column: 27FI0043:27FC0043.MEAS Bad: 115 I/O Timeout: 1 Column: 27FI0063:27FC0063.MEAS Bad: 8 I/O Timeout: 1 Column: 27LI0475:27LA0475.PNT I/O Timeout: 1 Column: 28FC0011:28FC0011.MEAS I/O Timeout: 1 Column: 28IT0013:28IT164A.PNT Not Connect: 2 I/O Timeout: 1 Column: 28LC1128:28LA1128.MEAS I/O Timeout: 1 Column: 28LI0756:28LI0756.PNT Not Connect: 2 I/O Timeout: 1 Column: 29FC0075:29FC0075.MEAS Bad: 1 Column: 29FC0078:29FC0078.MEAS Bad: 4 Column: 29FC0105:29FC0105.MEAS Bad: 29 Column: 29FC0114:29FC0114.MEAS Bad: 226 Column: 29JC0023:29JA0023.PNT Bad: 4 Column: 29JC0024:29JA0024.PNT Bad: 5 Column: 29JC133A:29JA133A.PNT Bad: 13 Column: 29LC0134:29LC0134.MEAS Bad: 2 Column: 29NC0007:29NC0007.MEAS Bad: 1 Column: 29PI0021:29PA0021.PNT Bad: 1 Column: 29PI0202:29PI0202.MEAS Bad: 1 Column: 29PT0326:29PA0326.PNT Bad: 2 Column: 32FC0817:32FB0817.OUTa Calc Failed: 5 Column: 32FI0097:32FA0097.PNT Bad: 1047 Column: 32FI0610:32FA0610.PNT Bad: 79 Column: 32II0439A:32IA0439A.PNT Bad: 5 Column: 32LC0056:32LC0056.MEAS Bad: 4 Column: 32LC0061:32LC0061.MEAS Bad: 3 Column: 32RL1BWTCD Bad: 4 Column: 32RL1COLaACT Bad: 4 Column: 32RL1MOICD Bad: 4 Column: 32RLBWTLSA Bad: 4 Column: 32RLMOILSA Bad: 4 Column: 32ZI0350:32ZA0350.PNT Bad: 326 Column: 38FC0050:38FB0050.RO01 I/O Timeout: 1 Column: 38FC0654:38FC0654.MEAS I/O Timeout: 1 Column: 39FC0071:39FA0071.MEAS I/O Timeout: 1 Column: 40_STEAM:1RVMTR_MW_AI.PNT I/O Timeout: 1 Column: 40FC0387:40FC0387.MEAS I/O Timeout: 1 Column: 40HORNS:MW_TEST.RO01 I/O Timeout: 1 Column: 40JI0333:40JI0333.PNT I/O Timeout: 1 Bad: 1 Column: 40PC0060:40PA0060.PNT I/O Timeout: 1 Column: 40PC0060:40PC0052.MEAS I/O Timeout: 1 Column: 40TC0072:40TC0072.MEAS I/O Timeout: 1 Column: 40TC0074:40TC0074.MEAS I/O Timeout: 1 Column: 44FT0011:44FT0011.PNT I/O Timeout: 1 Column: 44LI0002:44LI0002.PNT I/O Timeout: 1 Bad: 1 Column: PM2.TotalSweetnerFlow Calc Failed: 429 Column: 32LC0274:32LC0274.MEAS.12hrmin : 41 Bad Total: 9 Column: 32PI0934:32PA0934.PNT Bad: 1047 Column: 32TC15AR:32TC15AR.OUT.24hravg : 10 Column: 32LC0756:32LC0756.MEAS Bad: 188 Column: 29FI0018:29FI0018.PNT Not Connect: 2 Column: BRWN_DYE_LVL:BRWN_DYE_LVL.PNT Bad: 219 Column: 29FC0079:29FC0079.MEAS Bad: 106 Column: 29LC0134:29LC0134.MEAS.12hrmax : 3 Bad Total: 5 Column: 32PC0140:32PC0140.MEAS Bad: 692 Column: 32G558:558TCV1.OUT Out of Serv: 26 Configure: 101 I/O Timeout: 59 Column: 32G558:558TCV1.OUT.24hravg Bad Total: 179 Column: 32FI0650:32FA0650.PNT Bad: 1 Column: 32FI0648:32FA0648.PNT Bad: 2 Column: 32LC0119:32LC0119.MEAS.12hrmax : 16 Column: 29II0001A:32IA454A.PNT Bad: 26 Column: LFStorageChestOutletFlow_CALC Calc Failed: 106 Column: 32FC0817:32FB0817.OUT Bad: 5 Column: 27FI0411:27FA0411.PNT Not Connect: 2 I/O Timeout: 1 Column: 28LC0678:28LC0678.MEAS Not Connect: 2 I/O Timeout: 1 Column: 32RLPRODWT Bad: 4 Column: 28FC1030:28FH1030.RI02 I/O Timeout: 1 Column: 28SI0679:28SI0679.PNT Not Connect: 2 I/O Timeout: 1 Column: 28FC1141:28FG1141.PNT I/O Timeout: 1 Bad: 1 Column: 28FC0732:28FC0732.MEAS Not Connect: 2 I/O Timeout: 1 Column: 27NC0021:27NC0021.MEAS I/O Timeout: 1 Column: 27FI0023:27FI0023.PNT Bad: 155 I/O Timeout: 1 Column: 27FI0043:27FI0043.PNT Bad: 115 I/O Timeout: 1 Column: 27NC0061:27NC0061.MEAS I/O Timeout: 1 Column: 27FI0063:27FI0063.PNT Bad: 9 I/O Timeout: 1 Column: 18NC0870:18NC0870.MEAS I/O Timeout: 1 Column: 18NC0807:18NC0807.MEAS Bad: 55 I/O Timeout: 1 Column: 18NC0833:18NA0833.PNT I/O Timeout: 1 Column: 18FC0873:18FC0873.MEAS I/O Timeout: 1 Column: 18FC0801:18FC0801.MEAS I/O Timeout: 1 Column: 18LC0020:18FD0020.RO0001 I/O Timeout: 1 Column: 18LC0806:18LA0806.PNT I/O Timeout: 1 Column: 18LC0284:18LA0284.PNT I/O Timeout: 1 Column: 28NC0926:28NC0926.MEAS I/O Timeout: 1 Column: 28NC0008:28NC0008.MEAS I/O Timeout: 1 Column: Crse_Scrn.Total Calc Failed: 1 Column: 28LC0010:28LC0010.MEAS I/O Timeout: 1 Column: 28LI0634:28LI0634.PNT Not Connect: 2 I/O Timeout: 1 Column: 28LC0043:28LC0043.MEAS I/O Timeout: 1 Column: PM2SteamKpphTon Calc Failed: 22 Column: 40PC0063:40PC0063.MEAS I/O Timeout: 1 Column: 40MAINS:40_0102.PNT_4 I/O Timeout: 1 Column: 40MAINS:40_0102.PNT_3 I/O Timeout: 1 Column: 32DRVMST_RD3:1STDRYER_DRW.MEAS Bad: 1 Column: 32DRVMST_RD3:2NDDRYER_DRW.MEAS Bad: 1 Column: 32DRVMST_RD5:CAL_DRW.MEAS Bad: 4 Column: 32DRVMST_RD6:REELDRUM_DRW.MEAS Bad: 5 Column: 12FI0110:12FI0110.PNT Bad: 52
Out[11]:
| 18LI0028:18LA0028.PNT | 19FC0002:19FC0002.MEAS | 19LI0472:19LI0472.PNT | 19LI0477:19LA0477.PNT | 22AUTOMAX_RD:1STDRYER_DFB.PNT | 22AUTOMAX_RD:REELDRUM_SFB.PNT | 27FI0023:27FC0023.MEAS | 27FI0043:27FC0043.MEAS | 27FI0063:27FC0063.MEAS | 27LI0475:27LA0475.PNT | ... | 28LC0043:28LC0043.MEAS | PM2SteamKpphTon | 40PC0063:40PC0063.MEAS | 40MAINS:40_0102.PNT_4 | 40MAINS:40_0102.PNT_3 | 32DRVMST_RD3:1STDRYER_DRW.MEAS | 32DRVMST_RD3:2NDDRYER_DRW.MEAS | 32DRVMST_RD5:CAL_DRW.MEAS | 32DRVMST_RD6:REELDRUM_DRW.MEAS | 12FI0110:12FI0110.PNT | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 69.30485 | 0.000701 | 88.91846 | 39.33166 | 60.34387 | 3252.165 | 1915.491 | 1916.352 | 2018.092 | 95.4622 | ... | 59.83677 | 3.159816 | 92.9706 | 715.5654 | 174.2866 | 29.78587 | 8.200021 | 3.712504 | NaN | 5478.449962 |
| 1 | 67.75152 | 0.000701 | 87.77137 | 39.16823 | 59.70235 | 3254.244 | 2017.103 | 2008.514 | 1999.071 | 89.0742 | ... | 60.81561 | 3.214383 | 91.48096 | 715.3773 | 174.9358 | 30.4474 | 8.18878 | 3.2 | NaN | 5383.978364 |
| 2 | 58.73729 | 0.000701 | 88.66229 | 39.07338 | 58.69795 | 3251.63 | 2502.759 | 2574.432 | 2599.252 | 80.99429 | ... | 60.15176 | 3.208416 | 90.2299 | 729.7198 | 165.1357 | 30.69206 | 8.699999 | 3.577793 | NaN | 5445.179291 |
| 3 | 70.72887 | 0.000701 | 84.09638 | 37.36133 | 54.17207 | 3250.593 | 2004.934 | 1789.857 | 1791.822 | 97.67078 | ... | 61.00882 | 3.278652 | 87.89799 | 733.9852 | 155.1065 | 32.06769 | 8.474746 | 3.2 | NaN | 5740.561517 |
| 4 | 66.93857 | 0.000701 | 85.3093 | 37.31067 | 54.43218 | 3251.013 | 1972.268 | 1784.242 | 1820.146 | 98.85621 | ... | 60.21341 | 3.270396 | 90.29742 | 727.0026 | 166.0546 | 32.28739 | 8.588867 | 3.483357 | NaN | 5496.862867 |
5 rows × 104 columns
In [13]:
summary_list = []
columns_after_4th = df_clean.iloc[:, 4:].columns
# Calculate 'Exception', NA (including empty strings), and their counts and percentages for each column
for col in columns_after_4th:
total_rows = len(df_clean)
exception_count = (df_clean[col] == 'Exception').sum()
exception_percentage = (exception_count / total_rows) * 100
na_count = df_clean[col].isna().sum() + (df_clean[col] == '').sum() + (df_clean[col] == 'nan').sum() # NA includes empty strings
na_percentage = (na_count / total_rows) * 100
# Calculate combined count and percentage for 'Exception' and 'NA'
combined_count = exception_count + na_count
combined_percentage = (combined_count / total_rows) * 100
# Append the results as a dictionary to the list
summary_list.append({
"Column": col,
"Exception Count": exception_count,
"Exception Percentage": exception_percentage,
"NA Count": na_count,
"NA Percentage": na_percentage,
"Combined Count": combined_count,
"Combined Percentage": combined_percentage
})
# Convert the list of dictionaries to a DataFrame
summary_df = pd.DataFrame(summary_list)
# Sort the DataFrame by 'Exception Percentage' in descending order
summary_df.sort_values(by="Combined Percentage", ascending=False, inplace=True)
summary_df.reset_index(drop=True)
columns_above_threshold = summary_df[summary_df['Combined Percentage'] > 30]['Column']
print("Columns with Combined Percentage greater than 30%:")
print(columns_above_threshold.tolist())
Columns with Combined Percentage greater than 30%: ['32FI0097:32FA0097.PNT', '32DPC613:32DPC613.MEAS', '32PI0934:32PA0934.PNT', 'PM2.TotalSweetnerFlow', '32PC0140:32PC0140.MEAS', '19LI0477:19LA0477.PNT', '32LC0756:32LC0756.MEAS']
In [15]:
# Step 1: Remove columns where Combined Percentage > 30%
columns_above_threshold = summary_df[summary_df['Combined Percentage'] > 30]['Column']
# df_clean.drop(columns=columns_above_threshold, axis=1, inplace=True)
# Step 2: Convert 'Exception' to NaN
df_clean.iloc[:, 4:] = df_clean.iloc[:, 4:].apply(pd.to_numeric, errors='coerce')
# Step 3: Check if all columns are now numeric
object_columns = df_clean.iloc[:, 4:].select_dtypes(include=['object']).columns
if len(object_columns) == 0:
print("All columns are now numeric.")
else:
print("The following columns are still not numeric:", object_columns.tolist())
The following columns are still not numeric: ['18LI0028:18LA0028.PNT', '19FC0002:19FC0002.MEAS', '19LI0472:19LI0472.PNT', '19LI0477:19LA0477.PNT', '22AUTOMAX_RD:1STDRYER_DFB.PNT', '22AUTOMAX_RD:REELDRUM_SFB.PNT', '27FI0023:27FC0023.MEAS', '27FI0043:27FC0043.MEAS', '27FI0063:27FC0063.MEAS', '27LI0475:27LA0475.PNT', '28FC0011:28FC0011.MEAS', '28IT0013:28IT164A.PNT', '28LC1128:28LA1128.MEAS', '28LI0756:28LI0756.PNT', '29FC0075:29FC0075.MEAS', '29FC0078:29FC0078.MEAS', '29FC0105:29FC0105.MEAS', '29FC0114:29FC0114.MEAS', '29JC0023:29JA0023.PNT', '29JC0024:29JA0024.PNT', '29JC133A:29JA133A.PNT', '29LC0134:29LC0134.MEAS', '29NC0007:29NC0007.MEAS', '29PI0021:29PA0021.PNT', '29PI0202:29PI0202.MEAS', '29PT0326:29PA0326.PNT', '32FC0817:32FB0817.OUTa', '32FI0097:32FA0097.PNT', '32FI0610:32FA0610.PNT', '32II0439A:32IA0439A.PNT', '32LC0056:32LC0056.MEAS', '32LC0061:32LC0061.MEAS', '32RL1BWTCD', '32RL1COLaACT', '32RL1MOICD', '32RLBWTLSA', '32RLMOILSA', '32ZI0350:32ZA0350.PNT', '38FC0050:38FB0050.RO01', '38FC0654:38FC0654.MEAS', '39FC0071:39FA0071.MEAS', '40_STEAM:1RVMTR_MW_AI.PNT', '40FC0387:40FC0387.MEAS', '40HORNS:MW_TEST.RO01', '40JI0333:40JI0333.PNT', '40PC0060:40PA0060.PNT', '40PC0060:40PC0052.MEAS', '40TC0072:40TC0072.MEAS', '40TC0074:40TC0074.MEAS', '44FT0011:44FT0011.PNT', '44LI0002:44LI0002.PNT', 'PM2.TotalSweetnerFlow', '32LC0274:32LC0274.MEAS.12hrmin', '32PI0934:32PA0934.PNT', '32TC15AR:32TC15AR.OUT.24hravg', '32LC0756:32LC0756.MEAS', '29FI0018:29FI0018.PNT', 'BRWN_DYE_LVL:BRWN_DYE_LVL.PNT', '29FC0079:29FC0079.MEAS', '29LC0134:29LC0134.MEAS.12hrmax', '32PC0140:32PC0140.MEAS', '32G558:558TCV1.OUT', '32G558:558TCV1.OUT.24hravg', '32FI0650:32FA0650.PNT', '32FI0648:32FA0648.PNT', '32LC0119:32LC0119.MEAS.12hrmax', '29II0001A:32IA454A.PNT', 'LFStorageChestOutletFlow_CALC', '32FC0817:32FB0817.OUT', '27FI0411:27FA0411.PNT', '28LC0678:28LC0678.MEAS', '32RLPRODWT', '28FC1030:28FH1030.RI02', '28SI0679:28SI0679.PNT', '28FC1141:28FG1141.PNT', '28FC0732:28FC0732.MEAS', '27NC0021:27NC0021.MEAS', '27FI0023:27FI0023.PNT', '27FI0043:27FI0043.PNT', '27NC0061:27NC0061.MEAS', '27FI0063:27FI0063.PNT', '18NC0870:18NC0870.MEAS', '18NC0807:18NC0807.MEAS', '18NC0833:18NA0833.PNT', '18FC0873:18FC0873.MEAS', '18FC0801:18FC0801.MEAS', '18LC0020:18FD0020.RO0001', '18LC0806:18LA0806.PNT', '18LC0284:18LA0284.PNT', '28NC0926:28NC0926.MEAS', '28NC0008:28NC0008.MEAS', 'Crse_Scrn.Total', '28LC0010:28LC0010.MEAS', '28LI0634:28LI0634.PNT', '28LC0043:28LC0043.MEAS', 'PM2SteamKpphTon', '40PC0063:40PC0063.MEAS', '40MAINS:40_0102.PNT_4', '40MAINS:40_0102.PNT_3', '32DRVMST_RD3:1STDRYER_DRW.MEAS', '32DRVMST_RD3:2NDDRYER_DRW.MEAS', '32DRVMST_RD5:CAL_DRW.MEAS', '32DRVMST_RD6:REELDRUM_DRW.MEAS', '12FI0110:12FI0110.PNT']
In [17]:
# Step 1: Remove columns where Combined Percentage > 30%
columns_above_threshold = summary_df[summary_df['Combined Percentage'] > 30]['Column']
# df_clean.drop(columns=columns_above_threshold, axis=1, inplace=True)
# Step 2: Convert 'Exception' to NaN
df_clean.iloc[:, 4:] = df_clean.iloc[:, 4:].apply(pd.to_numeric, errors='coerce')
# Step 3: Check if all columns are now numeric
object_columns = df_clean.iloc[:, 4:].select_dtypes(include=['object']).columns
if len(object_columns) == 0:
print("All columns are now numeric.")
else:
print("The following columns are still not numeric:", object_columns.tolist())
The following columns are still not numeric: ['18LI0028:18LA0028.PNT', '19FC0002:19FC0002.MEAS', '19LI0472:19LI0472.PNT', '19LI0477:19LA0477.PNT', '22AUTOMAX_RD:1STDRYER_DFB.PNT', '22AUTOMAX_RD:REELDRUM_SFB.PNT', '27FI0023:27FC0023.MEAS', '27FI0043:27FC0043.MEAS', '27FI0063:27FC0063.MEAS', '27LI0475:27LA0475.PNT', '28FC0011:28FC0011.MEAS', '28IT0013:28IT164A.PNT', '28LC1128:28LA1128.MEAS', '28LI0756:28LI0756.PNT', '29FC0075:29FC0075.MEAS', '29FC0078:29FC0078.MEAS', '29FC0105:29FC0105.MEAS', '29FC0114:29FC0114.MEAS', '29JC0023:29JA0023.PNT', '29JC0024:29JA0024.PNT', '29JC133A:29JA133A.PNT', '29LC0134:29LC0134.MEAS', '29NC0007:29NC0007.MEAS', '29PI0021:29PA0021.PNT', '29PI0202:29PI0202.MEAS', '29PT0326:29PA0326.PNT', '32FC0817:32FB0817.OUTa', '32FI0097:32FA0097.PNT', '32FI0610:32FA0610.PNT', '32II0439A:32IA0439A.PNT', '32LC0056:32LC0056.MEAS', '32LC0061:32LC0061.MEAS', '32RL1BWTCD', '32RL1COLaACT', '32RL1MOICD', '32RLBWTLSA', '32RLMOILSA', '32ZI0350:32ZA0350.PNT', '38FC0050:38FB0050.RO01', '38FC0654:38FC0654.MEAS', '39FC0071:39FA0071.MEAS', '40_STEAM:1RVMTR_MW_AI.PNT', '40FC0387:40FC0387.MEAS', '40HORNS:MW_TEST.RO01', '40JI0333:40JI0333.PNT', '40PC0060:40PA0060.PNT', '40PC0060:40PC0052.MEAS', '40TC0072:40TC0072.MEAS', '40TC0074:40TC0074.MEAS', '44FT0011:44FT0011.PNT', '44LI0002:44LI0002.PNT', 'PM2.TotalSweetnerFlow', '32LC0274:32LC0274.MEAS.12hrmin', '32PI0934:32PA0934.PNT', '32TC15AR:32TC15AR.OUT.24hravg', '32LC0756:32LC0756.MEAS', '29FI0018:29FI0018.PNT', 'BRWN_DYE_LVL:BRWN_DYE_LVL.PNT', '29FC0079:29FC0079.MEAS', '29LC0134:29LC0134.MEAS.12hrmax', '32PC0140:32PC0140.MEAS', '32G558:558TCV1.OUT', '32G558:558TCV1.OUT.24hravg', '32FI0650:32FA0650.PNT', '32FI0648:32FA0648.PNT', '32LC0119:32LC0119.MEAS.12hrmax', '29II0001A:32IA454A.PNT', 'LFStorageChestOutletFlow_CALC', '32FC0817:32FB0817.OUT', '27FI0411:27FA0411.PNT', '28LC0678:28LC0678.MEAS', '32RLPRODWT', '28FC1030:28FH1030.RI02', '28SI0679:28SI0679.PNT', '28FC1141:28FG1141.PNT', '28FC0732:28FC0732.MEAS', '27NC0021:27NC0021.MEAS', '27FI0023:27FI0023.PNT', '27FI0043:27FI0043.PNT', '27NC0061:27NC0061.MEAS', '27FI0063:27FI0063.PNT', '18NC0870:18NC0870.MEAS', '18NC0807:18NC0807.MEAS', '18NC0833:18NA0833.PNT', '18FC0873:18FC0873.MEAS', '18FC0801:18FC0801.MEAS', '18LC0020:18FD0020.RO0001', '18LC0806:18LA0806.PNT', '18LC0284:18LA0284.PNT', '28NC0926:28NC0926.MEAS', '28NC0008:28NC0008.MEAS', 'Crse_Scrn.Total', '28LC0010:28LC0010.MEAS', '28LI0634:28LI0634.PNT', '28LC0043:28LC0043.MEAS', 'PM2SteamKpphTon', '40PC0063:40PC0063.MEAS', '40MAINS:40_0102.PNT_4', '40MAINS:40_0102.PNT_3', '32DRVMST_RD3:1STDRYER_DRW.MEAS', '32DRVMST_RD3:2NDDRYER_DRW.MEAS', '32DRVMST_RD5:CAL_DRW.MEAS', '32DRVMST_RD6:REELDRUM_DRW.MEAS', '12FI0110:12FI0110.PNT']
In [19]:
summary_list = []
columns_after_4th = df_clean.iloc[:, 4:].columns
# Calculate 'Exception', NA (including empty strings), and their counts and percentages for each column
for col in columns_after_4th:
total_rows = len(df_clean)
exception_count = (df_clean[col] == 'Exception').sum()
exception_percentage = (exception_count / total_rows) * 100
na_count = df_clean[col].isna().sum() + (df_clean[col] == '').sum() + (df_clean[col] == 'nan').sum() # NA includes empty strings
na_percentage = (na_count / total_rows) * 100
# Calculate combined count and percentage for 'Exception' and 'NA'
combined_count = exception_count + na_count
combined_percentage = (combined_count / total_rows) * 100
# Append the results as a dictionary to the list
summary_list.append({
"Column": col,
"Exception Count": exception_count,
"Exception Percentage": exception_percentage,
"NA Count": na_count,
"NA Percentage": na_percentage,
"Combined Count": combined_count,
"Combined Percentage": combined_percentage
})
# Convert the list of dictionaries to a DataFrame
summary_df = pd.DataFrame(summary_list)
# Sort the DataFrame by 'Exception Percentage' in descending order
summary_df.sort_values(by="Combined Percentage", ascending=False, inplace=True)
summary_df.reset_index(drop=True)
Out[19]:
| Column | Exception Count | Exception Percentage | NA Count | NA Percentage | Combined Count | Combined Percentage | |
|---|---|---|---|---|---|---|---|
| 0 | 32FI0097:32FA0097.PNT | 0 | 0.0 | 2987 | 91.373509 | 2987 | 91.373509 |
| 1 | 32DPC613:32DPC613.MEAS | 0 | 0.0 | 1678 | 51.330682 | 1678 | 51.330682 |
| 2 | 32PI0934:32PA0934.PNT | 0 | 0.0 | 1665 | 50.933007 | 1665 | 50.933007 |
| 3 | PM2.TotalSweetnerFlow | 0 | 0.0 | 1513 | 46.283267 | 1513 | 46.283267 |
| 4 | 32PC0140:32PC0140.MEAS | 0 | 0.0 | 1234 | 37.748547 | 1234 | 37.748547 |
| ... | ... | ... | ... | ... | ... | ... | ... |
| 513 | 32G477:477PT1.PNT | 0 | 0.0 | 0 | 0.000000 | 0 | 0.000000 |
| 514 | 32PC0611:32PC0611.OUTAVG | 0 | 0.0 | 0 | 0.000000 | 0 | 0.000000 |
| 515 | 32LC0682:32LC0682.OUTAVG | 0 | 0.0 | 0 | 0.000000 | 0 | 0.000000 |
| 516 | 32LC0636:32LC0636.OUTAVG | 0 | 0.0 | 0 | 0.000000 | 0 | 0.000000 |
| 517 | 32LC0624:32LC0624.OUTAVG | 0 | 0.0 | 0 | 0.000000 | 0 | 0.000000 |
518 rows × 7 columns
In [151]:
import pandas as pd
# Load the main dataset containing sensor data as columns
columns = df.columns[4:]
# Load the sensor description file
file_path = 'SmurfitWestrockConfidential_SensorDescription.xlsx'
sensor_data = pd.read_excel(file_path)
# Separate the sensor ID and description for better classification
sensor_data.columns = ['Sensor ID', 'Description']
# Keywords for different types of sensors
temperature_keywords = ["Temperature", "Temp", "Heat"]
pressure_keywords = ["Pressure", "Press", "Psi"]
flow_keywords = ["Flow", "Rate", "GPM"]
speed_keywords = ["Speed", "RPM", "Velocity"]
tank_level_keywords = ["Level", "Tank", "Height", "Volume", "Saveall", "Pct", "level", "%"]
water_keywords = ["Water", "Aqua", "H2O"]
dryer_keywords = ["Dryer", "Drying"]
steam_keywords = ["Steam", "Hdr", "Section"]
control_keywords = ["Control", "Valve", "Position", "Regulator"]
vacuum_keywords = ["Vacuum", "Suction"]
agitator_pulper_keywords = ["Pulper", "Agitator"]
blend_keywords = ["Blend", "Stock", "Chest"]
output_keywords = ["Output", "Out"]
average_keywords = ["Average", "Avg"]
refiner_keywords = ["Refiner"]
reel_keywords = ["Reel"]
roll_keywords = ["Roll"]
weight_keywords = ["lb", "ton"]
pump_keywords = ["Pump"]
box_keywords = ["Box"]
screen_keywords = ["Screen"]
receiver_keywords = ["Receiver", "Vessel", "Silo"]
aid_keywords = ["Aid"]
shower_keywords = ["Shower"]
# Classify sensors into different types based on keywords
def classify_sensors(sensor_data, keywords):
return sensor_data[sensor_data['Description'].str.contains('|'.join(keywords), case=False, na=False)]['Sensor ID'].tolist()
# Extract sensor IDs for each category
temperature_ids = classify_sensors(sensor_data, temperature_keywords)
pressure_ids = classify_sensors(sensor_data, pressure_keywords)
flow_ids = classify_sensors(sensor_data, flow_keywords)
speed_ids = classify_sensors(sensor_data, speed_keywords)
tank_level_ids = classify_sensors(sensor_data, tank_level_keywords)
water_ids = classify_sensors(sensor_data, water_keywords)
dryer_ids = classify_sensors(sensor_data, dryer_keywords)
steam_ids = classify_sensors(sensor_data, steam_keywords)
control_ids = classify_sensors(sensor_data, control_keywords)
vacuum_ids = classify_sensors(sensor_data, vacuum_keywords)
agitator_pulper_ids = classify_sensors(sensor_data, agitator_pulper_keywords)
blend_ids = classify_sensors(sensor_data, blend_keywords)
output_ids = classify_sensors(sensor_data, output_keywords)
average_ids = classify_sensors(sensor_data, average_keywords)
refiner_ids = classify_sensors(sensor_data, refiner_keywords)
reel_ids = classify_sensors(sensor_data, reel_keywords)
roll_ids = classify_sensors(sensor_data, roll_keywords)
weight_ids = classify_sensors(sensor_data, weight_keywords)
pump_ids = classify_sensors(sensor_data, pump_keywords)
box_ids = classify_sensors(sensor_data, box_keywords)
screen_ids = classify_sensors(sensor_data, screen_keywords)
receiver_ids = classify_sensors(sensor_data, receiver_keywords)
aid_ids = classify_sensors(sensor_data, aid_keywords)
shower_ids = classify_sensors(sensor_data, shower_keywords)
# Initialize classification dictionary for different sensor types
column_classification = {
'Temperature': [],
'Pressure': [],
'Flow': [],
'Speed': [],
'Tank Level': [],
'Water': [],
'Dryer': [],
'Steam': [],
'Control': [],
'Vacuum': [],
'Agitator & Pulper': [],
'Blend': [],
'Output': [],
'Average': [],
'Refiner': [],
'Reel': [],
'Roll': [],
'Weight': [],
'Pump': [],
'Box': [],
'Screen': [],
'Receiver': [],
'Aid': [],
'Shower': [],
}
# Classify columns based on sensor types
for col in columns:
if col in temperature_ids:
column_classification['Temperature'].append(col)
elif col in pressure_ids:
column_classification['Pressure'].append(col)
elif col in flow_ids:
column_classification['Flow'].append(col)
elif col in speed_ids:
column_classification['Speed'].append(col)
elif col in tank_level_ids:
column_classification['Tank Level'].append(col)
elif col in water_ids:
column_classification['Water'].append(col)
elif col in dryer_ids:
column_classification['Dryer'].append(col)
elif col in steam_ids:
column_classification['Steam'].append(col)
elif col in control_ids:
column_classification['Control'].append(col)
elif col in vacuum_ids:
column_classification['Vacuum'].append(col)
elif col in agitator_pulper_ids:
column_classification['Agitator & Pulper'].append(col)
elif col in blend_ids:
column_classification['Blend'].append(col)
elif col in output_ids:
column_classification['Output'].append(col)
elif col in average_ids:
column_classification['Average'].append(col)
elif col in refiner_ids:
column_classification['Refiner'].append(col)
elif col in reel_ids:
column_classification['Reel'].append(col)
elif col in roll_ids:
column_classification['Roll'].append(col)
elif col in weight_ids:
column_classification['Weight'].append(col)
elif col in pump_ids:
column_classification['Pump'].append(col)
elif col in box_ids:
column_classification['Box'].append(col)
elif col in screen_ids:
column_classification['Screen'].append(col)
elif col in receiver_ids:
column_classification['Receiver'].append(col)
elif col in aid_ids:
column_classification['Aid'].append(col)
elif col in shower_ids:
column_classification['Shower'].append(col)
else:
# If still ungrouped, classify as "Other Sensors"
if 'Other Sensors' not in column_classification:
column_classification['Other Sensors'] = []
column_classification['Other Sensors'].append(col)
# Output
# Calculate the count of columns in each category
column_counts = {key: len(value) for key, value in column_classification.items()}
# Display the count of columns for each category
print("Count of Columns in Each Category:")
for category, count in column_counts.items():
print(f"{category}: {count}")
Count of Columns in Each Category: Temperature: 31 Pressure: 82 Flow: 68 Speed: 6 Tank Level: 90 Water: 5 Dryer: 32 Steam: 32 Control: 7 Vacuum: 12 Agitator & Pulper: 13 Blend: 11 Output: 44 Average: 2 Refiner: 7 Reel: 8 Roll: 4 Weight: 6 Pump: 5 Box: 9 Screen: 3 Receiver: 10 Aid: 3 Shower: 3 Other Sensors: 25
In [152]:
average_columns = column_classification['Average']
print(average_columns)
['32RLBWTLSA', '32RLMOILSA']
In [153]:
df_clean['32RLBWTLSA'] = df_clean['32RLBWTLSA'].fillna(df_clean['32RLBWTLSA'].mean())
df_clean['32RLMOILSA'] = df_clean['32RLMOILSA'].fillna(df_clean['32RLMOILSA'].mean())
# Confirm that there are no more missing values in those two columns
df_clean[['32RLBWTLSA', '32RLMOILSA']].isnull().sum()
Out[153]:
32RLBWTLSA 0 32RLMOILSA 0 dtype: int64
In [154]:
# Function to process missing values for each group except the ones handled by average
def process_missing_values(category_name, category_columns):
if not category_columns:
print(f"\nNo columns found for category {category_name}. Skipping...")
return
df_category = df_clean[category_columns]
# Drop rows where all values are missing
df_category.dropna(how='all', inplace=True)
# Calculate the correlation matrix including missing values
correlation_matrix_with_missing = df_category.corr()
# Plot the correlation heatmap before processing missing data
plt.figure(figsize=(10, 8))
sns.heatmap(correlation_matrix_with_missing, annot=False, cmap="coolwarm", vmin=-1, vmax=1)
plt.title(f'Correlation Heatmap for {category_name} Group (Before Processing Missing Data)')
plt.show()
# Display the correlation matrix with missing data
print(f"\nCorrelation Matrix with Missing Data for {category_name}:")
print(correlation_matrix_with_missing)
# Create a temporary dataset for subsequent outlier detection and correlation analysis
df_temp = df_category.dropna()
# Check if there is enough data for analysis
if df_temp.shape[0] < 2:
print(f"Insufficient data to process category {category_name}. Skipping...")
return
# High correlation: Random Forest imputation
high_correlation_cols = correlation_matrix_with_missing[
(correlation_matrix_with_missing.abs() >= 0.7) & (correlation_matrix_with_missing.abs() < 1)
].columns
for col in high_correlation_cols:
if df_category[col].isnull().any():
correlated_cols = correlation_matrix_with_missing[col].abs().sort_values(ascending=False).index[1:]
correlated_cols = [c for c in correlated_cols if df_category[c].isnull().sum() == 0]
if len(correlated_cols) == 0:
print(f"No suitable correlated columns found for {col} to perform Random Forest imputation.")
continue
not_null_data = df_category[df_category[col].notnull()][[col] + correlated_cols]
X_train = not_null_data[correlated_cols]
y_train = not_null_data[col]
missing_data = df_category[df_category[col].isnull()][correlated_cols]
if missing_data.empty:
print(f"No missing data found for {col}.")
continue
rf_model = RandomForestRegressor(n_estimators=100, random_state=0)
rf_model.fit(X_train, y_train)
predicted_values = rf_model.predict(missing_data)
# Ensure the lengths match before assigning values
if len(predicted_values) == df_clean.loc[df_clean[col].isnull(), col].shape[0]:
df_clean.loc[df_clean[col].isnull(), col] = predicted_values
print(f"Filled missing values in {col} using Random Forest.")
else:
print(f"Length mismatch when trying to fill {col}. Skipping this column.")
# Medium correlation: Iterative Imputer (MICE)
medium_correlation_cols = correlation_matrix_with_missing[
(correlation_matrix_with_missing.abs() >= 0.3) & (correlation_matrix_with_missing.abs() < 0.7)
].columns
if len(medium_correlation_cols) > 0:
mice_imputer = IterativeImputer(random_state=0)
df_clean[medium_correlation_cols] = mice_imputer.fit_transform(df_clean[medium_correlation_cols])
print(f"Filled missing values in medium correlation columns using MICE: {list(medium_correlation_cols)}")
# Low correlation: Forward and backward fill
low_correlation_cols = correlation_matrix_with_missing[correlation_matrix_with_missing.abs() < 0.3].columns
for col in low_correlation_cols:
if df_category[col].isnull().any():
df_clean[col].fillna(method='ffill', inplace=True)
df_clean[col].fillna(method='bfill', inplace=True)
print(f"Filled missing values in {col} using forward and backward fill.")
# Process each category for missing value imputation
for category_name, category_columns in column_classification.items():
process_missing_values(category_name, category_columns)
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Temperature:
32G478:478TT1.PNT 32G478:478TT2.PNT \
32G478:478TT1.PNT 1.000000 0.264688
32G478:478TT2.PNT 0.264688 1.000000
32G478:478TT3.PNT 0.249901 0.866613
32G478:478TT4.PNT 0.185570 0.890143
32G478:478TT5.PNT 0.861548 0.200808
32G558:558TT1.PNT 0.022822 0.011545
32G568:568TT1.PNT 0.157260 0.266324
32TC0122:32TC0122.MEAS 0.412529 -0.100430
32TC1030:32TC1030.MEAS 0.024992 0.122899
32TI016A:32TA016A.PNT 0.512700 0.331925
40TC0072:40TC0072.MEAS 0.010496 0.428728
EMGT_XMTRS:TEMP_AMB.PNT -0.029918 0.209675
32TC0325:32TA0325.PNT -0.007560 0.263013
32TC0325:32TC0325.MEAS -0.007264 0.262880
32FC0620:32FB0620.OUT -0.473482 -0.053269
32TC0827:32TA0827.PNT 0.316223 0.393191
32TC0827:32TC0827.MEAS 0.315776 0.392702
32TC15AR:32TC15AR.OUT.24hravg 0.235453 -0.124611
32TC16AR:32TC16AR.OUT.24hravg 0.148825 0.044306
32TI16AA:32TI16AA.PNT 0.226541 0.185425
32TC0627:32TC0627.OUT -0.106715 -0.023544
32TC0641:32TC0641.MEAS 0.516999 0.197828
32G558:558TC1.OUT 0.212953 0.204540
32TC0627:32TC0627.MEAS 0.359817 0.251457
32TC1026:32TC1026.OUT 0.022504 0.164700
32TC0638:32TC0638.MEAS 0.342032 0.230893
32TC0640:32TC0640.MEAS 0.346562 0.118198
32TC0664:32TA0664.PNT 0.346262 0.117579
32TC0608:32TA0608.PNT -0.267775 0.194085
32TC0608:32TC0608.MEAS -0.267900 0.193959
40MAINS:40_0102.PNT_4 -0.056707 0.229878
32G478:478TT3.PNT 32G478:478TT4.PNT \
32G478:478TT1.PNT 0.249901 0.185570
32G478:478TT2.PNT 0.866613 0.890143
32G478:478TT3.PNT 1.000000 0.934133
32G478:478TT4.PNT 0.934133 1.000000
32G478:478TT5.PNT 0.177514 0.123481
32G558:558TT1.PNT 0.043614 0.057137
32G568:568TT1.PNT 0.208076 0.208746
32TC0122:32TC0122.MEAS -0.220137 -0.256841
32TC1030:32TC1030.MEAS 0.164957 0.204461
32TI016A:32TA016A.PNT 0.241687 0.277130
40TC0072:40TC0072.MEAS 0.455166 0.480139
EMGT_XMTRS:TEMP_AMB.PNT 0.235264 0.304341
32TC0325:32TA0325.PNT 0.291237 0.355272
32TC0325:32TC0325.MEAS 0.291026 0.355175
32FC0620:32FB0620.OUT 0.013787 0.013555
32TC0827:32TA0827.PNT 0.367299 0.402802
32TC0827:32TC0827.MEAS 0.366714 0.402259
32TC15AR:32TC15AR.OUT.24hravg -0.165581 -0.120347
32TC16AR:32TC16AR.OUT.24hravg -0.029380 0.028848
32TI16AA:32TI16AA.PNT 0.150157 0.177214
32TC0627:32TC0627.OUT -0.065400 -0.076596
32TC0641:32TC0641.MEAS 0.126242 0.117985
32G558:558TC1.OUT 0.160536 0.226821
32TC0627:32TC0627.MEAS 0.127666 0.122634
32TC1026:32TC1026.OUT 0.123753 0.089612
32TC0638:32TC0638.MEAS 0.201242 0.177510
32TC0640:32TC0640.MEAS 0.075111 0.055723
32TC0664:32TA0664.PNT 0.074606 0.055297
32TC0608:32TA0608.PNT 0.402992 0.369759
32TC0608:32TC0608.MEAS 0.402820 0.369632
40MAINS:40_0102.PNT_4 0.300767 0.327516
32G478:478TT5.PNT 32G558:558TT1.PNT \
32G478:478TT1.PNT 0.861548 0.022822
32G478:478TT2.PNT 0.200808 0.011545
32G478:478TT3.PNT 0.177514 0.043614
32G478:478TT4.PNT 0.123481 0.057137
32G478:478TT5.PNT 1.000000 0.039288
32G558:558TT1.PNT 0.039288 1.000000
32G568:568TT1.PNT 0.154435 0.005586
32TC0122:32TC0122.MEAS 0.440921 -0.075673
32TC1030:32TC1030.MEAS 0.078197 0.134735
32TI016A:32TA016A.PNT 0.624837 0.032455
40TC0072:40TC0072.MEAS -0.055128 0.034519
EMGT_XMTRS:TEMP_AMB.PNT -0.049780 0.115795
32TC0325:32TA0325.PNT -0.037238 0.144295
32TC0325:32TC0325.MEAS -0.036866 0.143762
32FC0620:32FB0620.OUT -0.563946 0.042787
32TC0827:32TA0827.PNT 0.256218 0.020512
32TC0827:32TC0827.MEAS 0.255901 0.019784
32TC15AR:32TC15AR.OUT.24hravg 0.316305 0.041679
32TC16AR:32TC16AR.OUT.24hravg 0.148215 0.010885
32TI16AA:32TI16AA.PNT 0.145541 -0.000968
32TC0627:32TC0627.OUT -0.076844 -0.022244
32TC0641:32TC0641.MEAS 0.608529 -0.006381
32G558:558TC1.OUT 0.186611 0.263402
32TC0627:32TC0627.MEAS 0.431415 -0.033792
32TC1026:32TC1026.OUT 0.038940 -0.016962
32TC0638:32TC0638.MEAS 0.376789 -0.004861
32TC0640:32TC0640.MEAS 0.426195 -0.028996
32TC0664:32TA0664.PNT 0.426034 -0.029060
32TC0608:32TA0608.PNT -0.342276 0.056039
32TC0608:32TC0608.MEAS -0.342398 0.056108
40MAINS:40_0102.PNT_4 -0.101311 0.047772
32G568:568TT1.PNT 32TC0122:32TC0122.MEAS \
32G478:478TT1.PNT 0.157260 0.412529
32G478:478TT2.PNT 0.266324 -0.100430
32G478:478TT3.PNT 0.208076 -0.220137
32G478:478TT4.PNT 0.208746 -0.256841
32G478:478TT5.PNT 0.154435 0.440921
32G558:558TT1.PNT 0.005586 -0.075673
32G568:568TT1.PNT 1.000000 0.038485
32TC0122:32TC0122.MEAS 0.038485 1.000000
32TC1030:32TC1030.MEAS -0.026881 -0.125620
32TI016A:32TA016A.PNT 0.037316 0.218212
40TC0072:40TC0072.MEAS -0.004806 -0.172958
EMGT_XMTRS:TEMP_AMB.PNT -0.011204 -0.236200
32TC0325:32TA0325.PNT -0.022141 -0.251562
32TC0325:32TC0325.MEAS -0.022137 -0.251152
32FC0620:32FB0620.OUT -0.051810 -0.299505
32TC0827:32TA0827.PNT 0.024129 -0.061100
32TC0827:32TC0827.MEAS 0.024245 -0.061308
32TC15AR:32TC15AR.OUT.24hravg 0.021769 0.329104
32TC16AR:32TC16AR.OUT.24hravg 0.013141 -0.058414
32TI16AA:32TI16AA.PNT 0.001146 0.168537
32TC0627:32TC0627.OUT -0.005671 -0.050583
32TC0641:32TC0641.MEAS 0.046507 0.170955
32G558:558TC1.OUT 0.003094 0.106704
32TC0627:32TC0627.MEAS 0.072016 0.128824
32TC1026:32TC1026.OUT 0.093820 0.104974
32TC0638:32TC0638.MEAS 0.070242 -0.050750
32TC0640:32TC0640.MEAS 0.052057 0.017186
32TC0664:32TA0664.PNT 0.052020 0.017135
32TC0608:32TA0608.PNT -0.027828 -0.447905
32TC0608:32TC0608.MEAS -0.027766 -0.447964
40MAINS:40_0102.PNT_4 0.018966 -0.245928
32TC1030:32TC1030.MEAS 32TI016A:32TA016A.PNT \
32G478:478TT1.PNT 0.024992 0.512700
32G478:478TT2.PNT 0.122899 0.331925
32G478:478TT3.PNT 0.164957 0.241687
32G478:478TT4.PNT 0.204461 0.277130
32G478:478TT5.PNT 0.078197 0.624837
32G558:558TT1.PNT 0.134735 0.032455
32G568:568TT1.PNT -0.026881 0.037316
32TC0122:32TC0122.MEAS -0.125620 0.218212
32TC1030:32TC1030.MEAS 1.000000 0.186614
32TI016A:32TA016A.PNT 0.186614 1.000000
40TC0072:40TC0072.MEAS 0.083084 0.004040
EMGT_XMTRS:TEMP_AMB.PNT 0.587192 0.071556
32TC0325:32TA0325.PNT 0.706769 0.090835
32TC0325:32TC0325.MEAS 0.706868 0.091275
32FC0620:32FB0620.OUT 0.076444 -0.614870
32TC0827:32TA0827.PNT 0.388079 0.287782
32TC0827:32TC0827.MEAS 0.388617 0.288290
32TC15AR:32TC15AR.OUT.24hravg 0.487532 0.319991
32TC16AR:32TC16AR.OUT.24hravg 0.033991 0.083554
32TI16AA:32TI16AA.PNT 0.451483 0.101481
32TC0627:32TC0627.OUT -0.084735 -0.120947
32TC0641:32TC0641.MEAS -0.012596 0.764767
32G558:558TC1.OUT 0.495615 0.233090
32TC0627:32TC0627.MEAS -0.123202 0.502114
32TC1026:32TC1026.OUT -0.112003 0.046535
32TC0638:32TC0638.MEAS -0.087018 0.441856
32TC0640:32TC0640.MEAS -0.140220 0.463075
32TC0664:32TA0664.PNT -0.140479 0.462988
32TC0608:32TA0608.PNT -0.018081 -0.224610
32TC0608:32TC0608.MEAS -0.017934 -0.224704
40MAINS:40_0102.PNT_4 -0.004212 -0.027189
... 32TC0641:32TC0641.MEAS 32G558:558TC1.OUT \
32G478:478TT1.PNT ... 0.516999 0.212953
32G478:478TT2.PNT ... 0.197828 0.204540
32G478:478TT3.PNT ... 0.126242 0.160536
32G478:478TT4.PNT ... 0.117985 0.226821
32G478:478TT5.PNT ... 0.608529 0.186611
32G558:558TT1.PNT ... -0.006381 0.263402
32G568:568TT1.PNT ... 0.046507 0.003094
32TC0122:32TC0122.MEAS ... 0.170955 0.106704
32TC1030:32TC1030.MEAS ... -0.012596 0.495615
32TI016A:32TA016A.PNT ... 0.764767 0.233090
40TC0072:40TC0072.MEAS ... -0.061144 0.118599
EMGT_XMTRS:TEMP_AMB.PNT ... -0.209647 0.506545
32TC0325:32TA0325.PNT ... -0.219582 0.617204
32TC0325:32TC0325.MEAS ... -0.219097 0.617226
32FC0620:32FB0620.OUT ... -0.793582 -0.011407
32TC0827:32TA0827.PNT ... 0.145963 0.392156
32TC0827:32TC0827.MEAS ... 0.146345 0.392311
32TC15AR:32TC15AR.OUT.24hravg ... 0.136178 0.594694
32TC16AR:32TC16AR.OUT.24hravg ... 0.108587 0.054475
32TI16AA:32TI16AA.PNT ... -0.118593 0.519691
32TC0627:32TC0627.OUT ... -0.026967 -0.211633
32TC0641:32TC0641.MEAS ... 1.000000 0.002234
32G558:558TC1.OUT ... 0.002234 1.000000
32TC0627:32TC0627.MEAS ... 0.690235 -0.114091
32TC1026:32TC1026.OUT ... 0.082091 -0.093317
32TC0638:32TC0638.MEAS ... 0.698657 -0.146489
32TC0640:32TC0640.MEAS ... 0.720530 -0.167254
32TC0664:32TA0664.PNT ... 0.720517 -0.167246
32TC0608:32TA0608.PNT ... -0.252245 -0.167702
32TC0608:32TC0608.MEAS ... -0.252289 -0.167672
40MAINS:40_0102.PNT_4 ... 0.005430 0.017753
32TC0627:32TC0627.MEAS 32TC1026:32TC1026.OUT \
32G478:478TT1.PNT 0.359817 0.022504
32G478:478TT2.PNT 0.251457 0.164700
32G478:478TT3.PNT 0.127666 0.123753
32G478:478TT4.PNT 0.122634 0.089612
32G478:478TT5.PNT 0.431415 0.038940
32G558:558TT1.PNT -0.033792 -0.016962
32G568:568TT1.PNT 0.072016 0.093820
32TC0122:32TC0122.MEAS 0.128824 0.104974
32TC1030:32TC1030.MEAS -0.123202 -0.112003
32TI016A:32TA016A.PNT 0.502114 0.046535
40TC0072:40TC0072.MEAS 0.278072 0.076188
EMGT_XMTRS:TEMP_AMB.PNT -0.250119 -0.155055
32TC0325:32TA0325.PNT -0.303698 -0.176969
32TC0325:32TC0325.MEAS -0.303341 -0.176498
32FC0620:32FB0620.OUT -0.518688 -0.061298
32TC0827:32TA0827.PNT 0.039769 -0.066235
32TC0827:32TC0827.MEAS 0.039634 -0.065645
32TC15AR:32TC15AR.OUT.24hravg -0.056892 -0.186310
32TC16AR:32TC16AR.OUT.24hravg 0.109558 0.022302
32TI16AA:32TI16AA.PNT -0.221974 -0.172065
32TC0627:32TC0627.OUT 0.279965 0.381715
32TC0641:32TC0641.MEAS 0.690235 0.082091
32G558:558TC1.OUT -0.114091 -0.093317
32TC0627:32TC0627.MEAS 1.000000 0.271814
32TC1026:32TC1026.OUT 0.271814 1.000000
32TC0638:32TC0638.MEAS 0.738527 0.324072
32TC0640:32TC0640.MEAS 0.595371 0.228750
32TC0664:32TA0664.PNT 0.595036 0.228228
32TC0608:32TA0608.PNT -0.234444 0.061077
32TC0608:32TC0608.MEAS -0.234418 0.061171
40MAINS:40_0102.PNT_4 0.258875 0.092972
32TC0638:32TC0638.MEAS 32TC0640:32TC0640.MEAS \
32G478:478TT1.PNT 0.342032 0.346562
32G478:478TT2.PNT 0.230893 0.118198
32G478:478TT3.PNT 0.201242 0.075111
32G478:478TT4.PNT 0.177510 0.055723
32G478:478TT5.PNT 0.376789 0.426195
32G558:558TT1.PNT -0.004861 -0.028996
32G568:568TT1.PNT 0.070242 0.052057
32TC0122:32TC0122.MEAS -0.050750 0.017186
32TC1030:32TC1030.MEAS -0.087018 -0.140220
32TI016A:32TA016A.PNT 0.441856 0.463075
40TC0072:40TC0072.MEAS 0.110708 -0.046370
EMGT_XMTRS:TEMP_AMB.PNT -0.217355 -0.223800
32TC0325:32TA0325.PNT -0.236063 -0.263878
32TC0325:32TC0325.MEAS -0.235871 -0.263217
32FC0620:32FB0620.OUT -0.521966 -0.582846
32TC0827:32TA0827.PNT 0.124201 0.086582
32TC0827:32TC0827.MEAS 0.124646 0.086680
32TC15AR:32TC15AR.OUT.24hravg -0.092949 -0.123028
32TC16AR:32TC16AR.OUT.24hravg 0.111539 0.129702
32TI16AA:32TI16AA.PNT -0.196258 -0.137914
32TC0627:32TC0627.OUT 0.273607 0.141503
32TC0641:32TC0641.MEAS 0.698657 0.720530
32G558:558TC1.OUT -0.146489 -0.167254
32TC0627:32TC0627.MEAS 0.738527 0.595371
32TC1026:32TC1026.OUT 0.324072 0.228750
32TC0638:32TC0638.MEAS 1.000000 0.722580
32TC0640:32TC0640.MEAS 0.722580 1.000000
32TC0664:32TA0664.PNT 0.722617 0.999838
32TC0608:32TA0608.PNT -0.051715 -0.176728
32TC0608:32TC0608.MEAS -0.051722 -0.176715
40MAINS:40_0102.PNT_4 0.202945 0.031483
32TC0664:32TA0664.PNT 32TC0608:32TA0608.PNT \
32G478:478TT1.PNT 0.346262 -0.267775
32G478:478TT2.PNT 0.117579 0.194085
32G478:478TT3.PNT 0.074606 0.402992
32G478:478TT4.PNT 0.055297 0.369759
32G478:478TT5.PNT 0.426034 -0.342276
32G558:558TT1.PNT -0.029060 0.056039
32G568:568TT1.PNT 0.052020 -0.027828
32TC0122:32TC0122.MEAS 0.017135 -0.447905
32TC1030:32TC1030.MEAS -0.140479 -0.018081
32TI016A:32TA016A.PNT 0.462988 -0.224610
40TC0072:40TC0072.MEAS -0.046465 0.258228
EMGT_XMTRS:TEMP_AMB.PNT -0.223775 0.074150
32TC0325:32TA0325.PNT -0.263935 0.107076
32TC0325:32TC0325.MEAS -0.263274 0.107033
32FC0620:32FB0620.OUT -0.582896 0.398106
32TC0827:32TA0827.PNT 0.086293 -0.111170
32TC0827:32TC0827.MEAS 0.086385 -0.110529
32TC15AR:32TC15AR.OUT.24hravg -0.122761 -0.406784
32TC16AR:32TC16AR.OUT.24hravg 0.129774 -0.099910
32TI16AA:32TI16AA.PNT -0.137664 -0.239101
32TC0627:32TC0627.OUT 0.141130 0.110365
32TC0641:32TC0641.MEAS 0.720517 -0.252245
32G558:558TC1.OUT -0.167246 -0.167702
32TC0627:32TC0627.MEAS 0.595036 -0.234444
32TC1026:32TC1026.OUT 0.228228 0.061077
32TC0638:32TC0638.MEAS 0.722617 -0.051715
32TC0640:32TC0640.MEAS 0.999838 -0.176728
32TC0664:32TA0664.PNT 1.000000 -0.176868
32TC0608:32TA0608.PNT -0.176868 1.000000
32TC0608:32TC0608.MEAS -0.176854 0.999986
40MAINS:40_0102.PNT_4 0.031796 0.215790
32TC0608:32TC0608.MEAS 40MAINS:40_0102.PNT_4
32G478:478TT1.PNT -0.267900 -0.056707
32G478:478TT2.PNT 0.193959 0.229878
32G478:478TT3.PNT 0.402820 0.300767
32G478:478TT4.PNT 0.369632 0.327516
32G478:478TT5.PNT -0.342398 -0.101311
32G558:558TT1.PNT 0.056108 0.047772
32G568:568TT1.PNT -0.027766 0.018966
32TC0122:32TC0122.MEAS -0.447964 -0.245928
32TC1030:32TC1030.MEAS -0.017934 -0.004212
32TI016A:32TA016A.PNT -0.224704 -0.027189
40TC0072:40TC0072.MEAS 0.258004 0.630679
EMGT_XMTRS:TEMP_AMB.PNT 0.074224 0.069907
32TC0325:32TA0325.PNT 0.107259 0.029841
32TC0325:32TC0325.MEAS 0.107215 0.030167
32FC0620:32FB0620.OUT 0.398248 0.058439
32TC0827:32TA0827.PNT -0.111119 0.007313
32TC0827:32TC0827.MEAS -0.110480 0.008305
32TC15AR:32TC15AR.OUT.24hravg -0.406665 -0.151442
32TC16AR:32TC16AR.OUT.24hravg -0.099802 -0.073473
32TI16AA:32TI16AA.PNT -0.239079 -0.111484
32TC0627:32TC0627.OUT 0.110347 0.134658
32TC0641:32TC0641.MEAS -0.252289 0.005430
32G558:558TC1.OUT -0.167672 0.017753
32TC0627:32TC0627.MEAS -0.234418 0.258875
32TC1026:32TC1026.OUT 0.061171 0.092972
32TC0638:32TC0638.MEAS -0.051722 0.202945
32TC0640:32TC0640.MEAS -0.176715 0.031483
32TC0664:32TA0664.PNT -0.176854 0.031796
32TC0608:32TA0608.PNT 0.999986 0.215790
32TC0608:32TC0608.MEAS 1.000000 0.215635
40MAINS:40_0102.PNT_4 0.215635 1.000000
[31 rows x 31 columns]
Filled missing values in 32TC1030:32TC1030.MEAS using Random Forest.
Filled missing values in 32TI016A:32TA016A.PNT using Random Forest.
Filled missing values in 40TC0072:40TC0072.MEAS using Random Forest.
Filled missing values in 32FC0620:32FB0620.OUT using Random Forest.
Filled missing values in 32TC0827:32TA0827.PNT using Random Forest.
Filled missing values in 32TC0827:32TC0827.MEAS using Random Forest.
Filled missing values in 32TC15AR:32TC15AR.OUT.24hravg using Random Forest.
Filled missing values in 32TI16AA:32TI16AA.PNT using Random Forest.
Filled missing values in 32TC0627:32TC0627.OUT using Random Forest.
Filled missing values in 32TC0641:32TC0641.MEAS using Random Forest.
Filled missing values in 32TC1026:32TC1026.OUT using Random Forest.
Filled missing values in 32TC0638:32TC0638.MEAS using Random Forest.
Filled missing values in 32TC0640:32TC0640.MEAS using Random Forest.
Filled missing values in 32TC0664:32TA0664.PNT using Random Forest.
Filled missing values in 32TC0608:32TA0608.PNT using Random Forest.
Filled missing values in 32TC0608:32TC0608.MEAS using Random Forest.
Filled missing values in 40MAINS:40_0102.PNT_4 using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32G478:478TT1.PNT', '32G478:478TT2.PNT', '32G478:478TT3.PNT', '32G478:478TT4.PNT', '32G478:478TT5.PNT', '32G558:558TT1.PNT', '32G568:568TT1.PNT', '32TC0122:32TC0122.MEAS', '32TC1030:32TC1030.MEAS', '32TI016A:32TA016A.PNT', '40TC0072:40TC0072.MEAS', 'EMGT_XMTRS:TEMP_AMB.PNT', '32TC0325:32TA0325.PNT', '32TC0325:32TC0325.MEAS', '32FC0620:32FB0620.OUT', '32TC0827:32TA0827.PNT', '32TC0827:32TC0827.MEAS', '32TC15AR:32TC15AR.OUT.24hravg', '32TC16AR:32TC16AR.OUT.24hravg', '32TI16AA:32TI16AA.PNT', '32TC0627:32TC0627.OUT', '32TC0641:32TC0641.MEAS', '32G558:558TC1.OUT', '32TC0627:32TC0627.MEAS', '32TC1026:32TC1026.OUT', '32TC0638:32TC0638.MEAS', '32TC0640:32TC0640.MEAS', '32TC0664:32TA0664.PNT', '32TC0608:32TA0608.PNT', '32TC0608:32TC0608.MEAS', '40MAINS:40_0102.PNT_4']
Filled missing values in 32TC1030:32TC1030.MEAS using forward and backward fill.
Filled missing values in 32TI016A:32TA016A.PNT using forward and backward fill.
Filled missing values in 40TC0072:40TC0072.MEAS using forward and backward fill.
Filled missing values in 32FC0620:32FB0620.OUT using forward and backward fill.
Filled missing values in 32TC0827:32TA0827.PNT using forward and backward fill.
Filled missing values in 32TC0827:32TC0827.MEAS using forward and backward fill.
Filled missing values in 32TC15AR:32TC15AR.OUT.24hravg using forward and backward fill.
Filled missing values in 32TI16AA:32TI16AA.PNT using forward and backward fill.
Filled missing values in 32TC0627:32TC0627.OUT using forward and backward fill.
Filled missing values in 32TC0641:32TC0641.MEAS using forward and backward fill.
Filled missing values in 32TC1026:32TC1026.OUT using forward and backward fill.
Filled missing values in 32TC0638:32TC0638.MEAS using forward and backward fill.
Filled missing values in 32TC0640:32TC0640.MEAS using forward and backward fill.
Filled missing values in 32TC0664:32TA0664.PNT using forward and backward fill.
Filled missing values in 32TC0608:32TA0608.PNT using forward and backward fill.
Filled missing values in 32TC0608:32TC0608.MEAS using forward and backward fill.
Filled missing values in 40MAINS:40_0102.PNT_4 using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Pressure:
29PI0014:29PA0014.PNT 29PI0015:29PA0015.PNT \
29PI0014:29PA0014.PNT 1.000000 0.893552
29PI0015:29PA0015.PNT 0.893552 1.000000
29PI0021:29PA0021.PNT 0.087895 0.066503
29PI0022:29PA0022.PNT 0.427440 0.397231
29PI0202:29PI0202.MEAS 0.086344 0.095587
... ... ...
32FC0820:32FA0820.PNT -0.758551 -0.605567
32PC0609:32PC0609.MEAS -0.018478 -0.031032
32FC0817:32FB0817.OUT -0.041204 0.064969
32PC0098:32PC0098.MEAS -0.050490 -0.043411
32DRVMST_RD2:SUCPRESS_CUR.MEAS 0.429112 0.473131
29PI0021:29PA0021.PNT 29PI0022:29PA0022.PNT \
29PI0014:29PA0014.PNT 0.087895 0.427440
29PI0015:29PA0015.PNT 0.066503 0.397231
29PI0021:29PA0021.PNT 1.000000 0.129202
29PI0022:29PA0022.PNT 0.129202 1.000000
29PI0202:29PI0202.MEAS -0.097092 0.042422
... ... ...
32FC0820:32FA0820.PNT 0.133043 -0.304637
32PC0609:32PC0609.MEAS 0.176008 -0.023177
32FC0817:32FB0817.OUT -0.040241 -0.078759
32PC0098:32PC0098.MEAS -0.457808 0.093314
32DRVMST_RD2:SUCPRESS_CUR.MEAS -0.258728 0.488382
29PI0202:29PI0202.MEAS 29PT0326:29PA0326.PNT \
29PI0014:29PA0014.PNT 0.086344 -0.024044
29PI0015:29PA0015.PNT 0.095587 0.034532
29PI0021:29PA0021.PNT -0.097092 0.155761
29PI0022:29PA0022.PNT 0.042422 0.190922
29PI0202:29PI0202.MEAS 1.000000 -0.065912
... ... ...
32FC0820:32FA0820.PNT -0.110029 0.220884
32PC0609:32PC0609.MEAS -0.010541 0.044024
32FC0817:32FB0817.OUT -0.036030 0.239287
32PC0098:32PC0098.MEAS 0.066593 -0.188401
32DRVMST_RD2:SUCPRESS_CUR.MEAS 0.148739 -0.132894
29PT0327:29PA0327.PNT 32FC0815:32FQ0815.RO02 \
29PI0014:29PA0014.PNT 0.138611 0.322686
29PI0015:29PA0015.PNT 0.128308 0.383781
29PI0021:29PA0021.PNT 0.033039 -0.264514
29PI0022:29PA0022.PNT 0.393670 0.375672
29PI0202:29PI0202.MEAS -0.028527 0.122612
... ... ...
32FC0820:32FA0820.PNT -0.092795 -0.276075
32PC0609:32PC0609.MEAS 0.003803 -0.072645
32FC0817:32FB0817.OUT 0.172155 -0.116726
32PC0098:32PC0098.MEAS 0.029752 0.529089
32DRVMST_RD2:SUCPRESS_CUR.MEAS 0.120653 0.655074
32FC0817:32FB0817.OUTa \
29PI0014:29PA0014.PNT -0.040305
29PI0015:29PA0015.PNT 0.065115
29PI0021:29PA0021.PNT -0.037997
29PI0022:29PA0022.PNT -0.077695
29PI0202:29PI0202.MEAS -0.035201
... ...
32FC0820:32FA0820.PNT 0.018145
32PC0609:32PC0609.MEAS -0.056320
32FC0817:32FB0817.OUT 0.998747
32PC0098:32PC0098.MEAS -0.169540
32DRVMST_RD2:SUCPRESS_CUR.MEAS -0.023981
32FC0817:32FQ0817.RO01 ... \
29PI0014:29PA0014.PNT -0.083411 ...
29PI0015:29PA0015.PNT -0.032694 ...
29PI0021:29PA0021.PNT -0.362944 ...
29PI0022:29PA0022.PNT 0.143347 ...
29PI0202:29PI0202.MEAS 0.059488 ...
... ... ...
32FC0820:32FA0820.PNT -0.008014 ...
32PC0609:32PC0609.MEAS -0.102597 ...
32FC0817:32FB0817.OUT 0.243573 ...
32PC0098:32PC0098.MEAS 0.536646 ...
32DRVMST_RD2:SUCPRESS_CUR.MEAS 0.344234 ...
32DC0679:32DC0679.OUT 32PI0913:32PA0913.PNT \
29PI0014:29PA0014.PNT 0.031454 0.114210
29PI0015:29PA0015.PNT 0.015677 0.056066
29PI0021:29PA0021.PNT -0.065849 0.287716
29PI0022:29PA0022.PNT 0.021603 0.156253
29PI0202:29PI0202.MEAS 0.032931 -0.093686
... ... ...
32FC0820:32FA0820.PNT -0.074849 -0.043097
32PC0609:32PC0609.MEAS 0.005873 0.026257
32FC0817:32FB0817.OUT 0.082435 0.149926
32PC0098:32PC0098.MEAS -0.070484 -0.308470
32DRVMST_RD2:SUCPRESS_CUR.MEAS 0.166717 -0.179851
32PI80AU:32PA80AU.PNT 32PC0153:32PC0153.OUT \
29PI0014:29PA0014.PNT 0.168871 -0.320134
29PI0015:29PA0015.PNT -0.000102 -0.278187
29PI0021:29PA0021.PNT 0.112610 -0.136709
29PI0022:29PA0022.PNT 0.153266 -0.230855
29PI0202:29PI0202.MEAS 0.049123 -0.072447
... ... ...
32FC0820:32FA0820.PNT -0.194472 0.272451
32PC0609:32PC0609.MEAS 0.023625 0.012572
32FC0817:32FB0817.OUT -0.275663 -0.082422
32PC0098:32PC0098.MEAS 0.113607 0.177805
32DRVMST_RD2:SUCPRESS_CUR.MEAS 0.419902 -0.060039
32FC0821:32FA0821.PNT 32FC0820:32FA0820.PNT \
29PI0014:29PA0014.PNT -0.041546 -0.758551
29PI0015:29PA0015.PNT -0.069158 -0.605567
29PI0021:29PA0021.PNT -0.294880 0.133043
29PI0022:29PA0022.PNT 0.184903 -0.304637
29PI0202:29PI0202.MEAS 0.080920 -0.110029
... ... ...
32FC0820:32FA0820.PNT -0.022432 1.000000
32PC0609:32PC0609.MEAS -0.051529 0.122913
32FC0817:32FB0817.OUT -0.478032 0.017981
32PC0098:32PC0098.MEAS 0.597266 -0.216214
32DRVMST_RD2:SUCPRESS_CUR.MEAS 0.324405 -0.435633
32PC0609:32PC0609.MEAS 32FC0817:32FB0817.OUT \
29PI0014:29PA0014.PNT -0.018478 -0.041204
29PI0015:29PA0015.PNT -0.031032 0.064969
29PI0021:29PA0021.PNT 0.176008 -0.040241
29PI0022:29PA0022.PNT -0.023177 -0.078759
29PI0202:29PI0202.MEAS -0.010541 -0.036030
... ... ...
32FC0820:32FA0820.PNT 0.122913 0.017981
32PC0609:32PC0609.MEAS 1.000000 -0.057521
32FC0817:32FB0817.OUT -0.057521 1.000000
32PC0098:32PC0098.MEAS -0.212244 -0.165698
32DRVMST_RD2:SUCPRESS_CUR.MEAS -0.061091 -0.021048
32PC0098:32PC0098.MEAS \
29PI0014:29PA0014.PNT -0.050490
29PI0015:29PA0015.PNT -0.043411
29PI0021:29PA0021.PNT -0.457808
29PI0022:29PA0022.PNT 0.093314
29PI0202:29PI0202.MEAS 0.066593
... ...
32FC0820:32FA0820.PNT -0.216214
32PC0609:32PC0609.MEAS -0.212244
32FC0817:32FB0817.OUT -0.165698
32PC0098:32PC0098.MEAS 1.000000
32DRVMST_RD2:SUCPRESS_CUR.MEAS 0.272181
32DRVMST_RD2:SUCPRESS_CUR.MEAS
29PI0014:29PA0014.PNT 0.429112
29PI0015:29PA0015.PNT 0.473131
29PI0021:29PA0021.PNT -0.258728
29PI0022:29PA0022.PNT 0.488382
29PI0202:29PI0202.MEAS 0.148739
... ...
32FC0820:32FA0820.PNT -0.435633
32PC0609:32PC0609.MEAS -0.061091
32FC0817:32FB0817.OUT -0.021048
32PC0098:32PC0098.MEAS 0.272181
32DRVMST_RD2:SUCPRESS_CUR.MEAS 1.000000
[82 rows x 82 columns]
Filled missing values in 29PI0021:29PA0021.PNT using Random Forest.
Filled missing values in 29PI0202:29PI0202.MEAS using Random Forest.
Filled missing values in 29PT0326:29PA0326.PNT using Random Forest.
Filled missing values in 29PT0327:29PA0327.PNT using Random Forest.
Filled missing values in 32FC0817:32FB0817.OUTa using Random Forest.
Filled missing values in 32FC0819:32FB0819.OUT using Random Forest.
Filled missing values in 32G481:481PCM4.OUT using Random Forest.
Filled missing values in 32PC0098:32PD0098.PNT using Random Forest.
Filled missing values in 32PI0199:32PC0199.MEAS using Random Forest.
Filled missing values in 32PI25AI:32PA25AI.PNT using Random Forest.
Filled missing values in 32LC0119:32FB0119.PNT using Random Forest.
Filled missing values in 32LC0119:32LC0119.OUT using Random Forest.
Filled missing values in 32DPC614:32DPC614.MEAS using Random Forest.
Filled missing values in 32PC0828:32PC0828.OUT using Random Forest.
Filled missing values in 32PI02AA:32PA02AA.PNT using Random Forest.
Filled missing values in 32LUBEOILRD1:FS_5F.PNT using Random Forest.
Filled missing values in 32PRBBOX:32PRBBOX.PNT using Random Forest.
Filled missing values in 29PC0077:29PC0077.OUT using Random Forest.
Filled missing values in 32DPC613:32DPC613.MEAS using Random Forest.
Filled missing values in 32DPC615:32DPC615.MEAS using Random Forest.
Filled missing values in 32PI0229:32PA0229.PNT using Random Forest.
Filled missing values in 32PI0910:32PA0910.PNT using Random Forest.
Filled missing values in 32LC0119:32LC0119.MEAS.12hrmax using Random Forest.
Filled missing values in 32PI0913:32PA0913.PNT using Random Forest.
Filled missing values in 32PC0153:32PC0153.OUT using Random Forest.
Filled missing values in 32FC0821:32FA0821.PNT using Random Forest.
Filled missing values in 32FC0820:32FA0820.PNT using Random Forest.
Filled missing values in 32PC0609:32PC0609.MEAS using Random Forest.
Filled missing values in 32FC0817:32FB0817.OUT using Random Forest.
Filled missing values in medium correlation columns using MICE: ['29PI0014:29PA0014.PNT', '29PI0015:29PA0015.PNT', '29PI0021:29PA0021.PNT', '29PI0022:29PA0022.PNT', '29PI0202:29PI0202.MEAS', '29PT0326:29PA0326.PNT', '29PT0327:29PA0327.PNT', '32FC0815:32FQ0815.RO02', '32FC0817:32FB0817.OUTa', '32FC0817:32FQ0817.RO01', '32FC0818:32FB0818.OUT', '32FC0819:32FB0819.OUT', '32FC0820:32FB0820.OUT', '32FC0821:32FB0821.OUT', '32G017:17PM2.NORM', '32G017:17PM2.OUT', '32G017:17PM3.OUT', '32G017:17PT1.PNT', '32G477:477PT1.PNT', '32G477:477PT2.PNT', '32G477:477PT3.PNT', '32G477:477PT4.PNT', '32G477:477PT5.PNT', '32G481:481PC1.MEAS', '32G481:481PCM4.OUT', '32G560:560PT1.PNT', '32PC0098:32PA0098.PNT', '32PC0098:32PD0098.PNT', '32PC0098:32PE0098.RO02', '32PC0139:32PC0139.MEAS', '32PC0147:32PC0147.MEAS', '32PC0153:32PC0153.MEAS', '32PC0681:32PB0681.OUT', '32PC0681:32PC0681.MEAS', '32PC15AO:32PC15AO.MEAS', '32PC15AO:32PC15AO.OUT', '32PC15AP:32PC15AP.MEAS', '32PC15AP:32PC15AP.OUT', '32PC15AQ:32PC15AQ.MEAS', '32PC15AQ:32PC15AQ.OUT', '32PC16AO:32PC16AO.MEAS', '32PC16AO:32PC16AO.OUT', '32PC16AP:32PC16AP.MEAS', '32PC16AP:32PC16AP.OUT', '32PC16AQ:32PC16AQ.MEAS', '32PC16AQ:32PC16AQ.OUT', '32PI0199:32PC0199.MEAS', '32PI0202:32PA0202.PNT', '32PI0296:32PA0296.PNT', '32PI0298:32PA0298.PNT', '32PI0855A:32PA0855A.PNT', '32PI25AI:32PA25AI.PNT', '32LC0119:32FB0119.PNT', '32LC0119:32LC0119.OUT', '32PI80AT:32PA80AT.PNT', '32DC0622:32DC0622.OUT', '32DPC614:32DPC614.MEAS', '32PC0828:32PB0828.OUT', '32PC0828:32PC0828.OUT', '32PI02AA:32PA02AA.PNT', '32LUBEOILRD1:FS_5F.PNT', '32PRBBOX:32PRBBOX.PNT', '29PC0077:29PC0077.OUT', '29PI0044:29PA0044.PNT', '32DPC613:32DPC613.MEAS', '32DPC615:32DPC615.MEAS', '32PI0229:32PA0229.PNT', '32DC0679:32DB0679.OUT', '32DI0059:32DA0059.PNT', '32PI0910:32PA0910.PNT', '32DC0670:32DB0670.OUT', '32LC0119:32LC0119.MEAS.12hrmax', '32DC0679:32DC0679.OUT', '32PI0913:32PA0913.PNT', '32PI80AU:32PA80AU.PNT', '32PC0153:32PC0153.OUT', '32FC0821:32FA0821.PNT', '32FC0820:32FA0820.PNT', '32PC0609:32PC0609.MEAS', '32FC0817:32FB0817.OUT', '32PC0098:32PC0098.MEAS', '32DRVMST_RD2:SUCPRESS_CUR.MEAS']
Filled missing values in 29PI0021:29PA0021.PNT using forward and backward fill.
Filled missing values in 29PI0202:29PI0202.MEAS using forward and backward fill.
Filled missing values in 29PT0326:29PA0326.PNT using forward and backward fill.
Filled missing values in 29PT0327:29PA0327.PNT using forward and backward fill.
Filled missing values in 32FC0817:32FB0817.OUTa using forward and backward fill.
Filled missing values in 32FC0819:32FB0819.OUT using forward and backward fill.
Filled missing values in 32G481:481PCM4.OUT using forward and backward fill.
Filled missing values in 32PC0098:32PD0098.PNT using forward and backward fill.
Filled missing values in 32PI0199:32PC0199.MEAS using forward and backward fill.
Filled missing values in 32PI25AI:32PA25AI.PNT using forward and backward fill.
Filled missing values in 32LC0119:32FB0119.PNT using forward and backward fill.
Filled missing values in 32LC0119:32LC0119.OUT using forward and backward fill.
Filled missing values in 32DPC614:32DPC614.MEAS using forward and backward fill.
Filled missing values in 32PC0828:32PC0828.OUT using forward and backward fill.
Filled missing values in 32PI02AA:32PA02AA.PNT using forward and backward fill.
Filled missing values in 32LUBEOILRD1:FS_5F.PNT using forward and backward fill.
Filled missing values in 32PRBBOX:32PRBBOX.PNT using forward and backward fill.
Filled missing values in 29PC0077:29PC0077.OUT using forward and backward fill.
Filled missing values in 32DPC613:32DPC613.MEAS using forward and backward fill.
Filled missing values in 32DPC615:32DPC615.MEAS using forward and backward fill.
Filled missing values in 32PI0229:32PA0229.PNT using forward and backward fill.
Filled missing values in 32PI0910:32PA0910.PNT using forward and backward fill.
Filled missing values in 32LC0119:32LC0119.MEAS.12hrmax using forward and backward fill.
Filled missing values in 32PI0913:32PA0913.PNT using forward and backward fill.
Filled missing values in 32PC0153:32PC0153.OUT using forward and backward fill.
Filled missing values in 32FC0821:32FA0821.PNT using forward and backward fill.
Filled missing values in 32FC0820:32FA0820.PNT using forward and backward fill.
Filled missing values in 32PC0609:32PC0609.MEAS using forward and backward fill.
Filled missing values in 32FC0817:32FB0817.OUT using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Flow:
27FI0023:27FC0023.MEAS 27FI0043:27FC0043.MEAS \
27FI0023:27FC0023.MEAS 1.000000 0.595799
27FI0043:27FC0043.MEAS 0.595799 1.000000
27FI0063:27FC0063.MEAS 0.352012 0.364718
28FC0011:28FC0011.MEAS 0.400291 0.182619
29FC0018:29FC0018.MEAS 0.140149 0.034809
... ... ...
18FC0801:18FC0801.MEAS 0.306309 0.232422
18LC0020:18FD0020.RO0001 0.144269 0.083416
19FC0206:19FG0206.RO04 0.137039 0.079182
Crse_Scrn.Total 0.361771 0.158196
12FI0110:12FI0110.PNT 0.021300 0.046573
27FI0063:27FC0063.MEAS 28FC0011:28FC0011.MEAS \
27FI0023:27FC0023.MEAS 0.352012 0.400291
27FI0043:27FC0043.MEAS 0.364718 0.182619
27FI0063:27FC0063.MEAS 1.000000 0.301546
28FC0011:28FC0011.MEAS 0.301546 1.000000
29FC0018:29FC0018.MEAS 0.160324 0.317316
... ... ...
18FC0801:18FC0801.MEAS 0.202552 0.097161
18LC0020:18FD0020.RO0001 0.110170 0.029857
19FC0206:19FG0206.RO04 0.136883 0.078187
Crse_Scrn.Total 0.243227 0.799860
12FI0110:12FI0110.PNT 0.022141 0.057338
29FC0018:29FC0018.MEAS 29FC0018:29FC0018.OUT \
27FI0023:27FC0023.MEAS 0.140149 0.066868
27FI0043:27FC0043.MEAS 0.034809 0.039214
27FI0063:27FC0063.MEAS 0.160324 0.131530
28FC0011:28FC0011.MEAS 0.317316 0.223884
29FC0018:29FC0018.MEAS 1.000000 0.614796
... ... ...
18FC0801:18FC0801.MEAS 0.053561 0.114247
18LC0020:18FD0020.RO0001 -0.002719 0.054356
19FC0206:19FG0206.RO04 -0.040862 0.057249
Crse_Scrn.Total 0.317501 0.239377
12FI0110:12FI0110.PNT -0.277948 -0.159948
29FC0019:29FC0019.MEAS 29FC0019:29FC0019.OUT \
27FI0023:27FC0023.MEAS 0.137126 -0.076681
27FI0043:27FC0043.MEAS 0.100600 -0.007131
27FI0063:27FC0063.MEAS 0.022814 -0.100726
28FC0011:28FC0011.MEAS 0.281303 -0.001560
29FC0018:29FC0018.MEAS 0.313218 -0.123795
... ... ...
18FC0801:18FC0801.MEAS -0.019742 -0.034091
18LC0020:18FD0020.RO0001 -0.060475 -0.037226
19FC0206:19FG0206.RO04 0.018286 0.028713
Crse_Scrn.Total 0.252848 -0.033684
12FI0110:12FI0110.PNT 0.077070 0.285752
29FC0075:29FC0075.MEAS 29FC0105:29FC0105.MEAS ... \
27FI0023:27FC0023.MEAS 0.137491 0.055524 ...
27FI0043:27FC0043.MEAS 0.038634 -0.029893 ...
27FI0063:27FC0063.MEAS 0.098916 0.064637 ...
28FC0011:28FC0011.MEAS 0.176961 0.203829 ...
29FC0018:29FC0018.MEAS 0.003749 0.236396 ...
... ... ... ...
18FC0801:18FC0801.MEAS 0.046115 0.047399 ...
18LC0020:18FD0020.RO0001 0.026783 0.024816 ...
19FC0206:19FG0206.RO04 0.098952 0.008516 ...
Crse_Scrn.Total 0.063164 0.131612 ...
12FI0110:12FI0110.PNT 0.261925 0.118412 ...
29FCA093:29FC0093.MEAS 32FC0425:32FA0425.PNT \
27FI0023:27FC0023.MEAS 0.352972 -0.026031
27FI0043:27FC0043.MEAS 0.140427 0.071655
27FI0063:27FC0063.MEAS 0.261647 0.093383
28FC0011:28FC0011.MEAS 0.733794 -0.067595
29FC0018:29FC0018.MEAS 0.464668 -0.064858
... ... ...
18FC0801:18FC0801.MEAS 0.082498 0.431487
18LC0020:18FD0020.RO0001 0.004612 0.567547
19FC0206:19FG0206.RO04 0.077864 0.473612
Crse_Scrn.Total 0.698180 -0.048095
12FI0110:12FI0110.PNT 0.002814 -0.119212
27FI0411:27FA0411.PNT 28FC1141:28FG1141.PNT \
27FI0023:27FC0023.MEAS 0.074014 0.021863
27FI0043:27FC0043.MEAS -0.004919 0.030034
27FI0063:27FC0063.MEAS 0.026617 0.070984
28FC0011:28FC0011.MEAS 0.253412 0.193428
29FC0018:29FC0018.MEAS -0.026894 -0.079507
... ... ...
18FC0801:18FC0801.MEAS 0.035982 0.011085
18LC0020:18FD0020.RO0001 0.046789 -0.001708
19FC0206:19FG0206.RO04 0.048751 0.002050
Crse_Scrn.Total 0.099893 -0.023291
12FI0110:12FI0110.PNT -0.099572 0.099568
28FC0732:28FC0732.MEAS 18FC0801:18FC0801.MEAS \
27FI0023:27FC0023.MEAS 0.090596 0.306309
27FI0043:27FC0043.MEAS -0.065101 0.232422
27FI0063:27FC0063.MEAS 0.078226 0.202552
28FC0011:28FC0011.MEAS 0.134898 0.097161
29FC0018:29FC0018.MEAS 0.075831 0.053561
... ... ...
18FC0801:18FC0801.MEAS 0.038761 1.000000
18LC0020:18FD0020.RO0001 0.048833 0.810072
19FC0206:19FG0206.RO04 0.099012 0.757878
Crse_Scrn.Total 0.083884 0.098397
12FI0110:12FI0110.PNT -0.033001 -0.052156
18LC0020:18FD0020.RO0001 19FC0206:19FG0206.RO04 \
27FI0023:27FC0023.MEAS 0.144269 0.137039
27FI0043:27FC0043.MEAS 0.083416 0.079182
27FI0063:27FC0063.MEAS 0.110170 0.136883
28FC0011:28FC0011.MEAS 0.029857 0.078187
29FC0018:29FC0018.MEAS -0.002719 -0.040862
... ... ...
18FC0801:18FC0801.MEAS 0.810072 0.757878
18LC0020:18FD0020.RO0001 1.000000 0.800408
19FC0206:19FG0206.RO04 0.800408 1.000000
Crse_Scrn.Total 0.032311 0.056966
12FI0110:12FI0110.PNT -0.064793 -0.010632
Crse_Scrn.Total 12FI0110:12FI0110.PNT
27FI0023:27FC0023.MEAS 0.361771 0.021300
27FI0043:27FC0043.MEAS 0.158196 0.046573
27FI0063:27FC0063.MEAS 0.243227 0.022141
28FC0011:28FC0011.MEAS 0.799860 0.057338
29FC0018:29FC0018.MEAS 0.317501 -0.277948
... ... ...
18FC0801:18FC0801.MEAS 0.098397 -0.052156
18LC0020:18FD0020.RO0001 0.032311 -0.064793
19FC0206:19FG0206.RO04 0.056966 -0.010632
Crse_Scrn.Total 1.000000 -0.081251
12FI0110:12FI0110.PNT -0.081251 1.000000
[68 rows x 68 columns]
Insufficient data to process category Flow. Skipping...
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Speed:
22AUTOMAX_RD:REELDRUM_SFB.PNT \
22AUTOMAX_RD:REELDRUM_SFB.PNT 1.000000
29SI072A:29SA072A.PNT 0.344038
29SI0112:29SA0112.PNT 0.073865
28SI0679:28SI0679.PNT 0.054864
32DRVMST_RD6:REELDRUM_SFB.MEAS 0.264184
32DRVMST_RD:COUCH_SFB.MEAS 0.264724
29SI072A:29SA072A.PNT 29SI0112:29SA0112.PNT \
22AUTOMAX_RD:REELDRUM_SFB.PNT 0.344038 0.073865
29SI072A:29SA072A.PNT 1.000000 0.109436
29SI0112:29SA0112.PNT 0.109436 1.000000
28SI0679:28SI0679.PNT 0.097129 0.110158
32DRVMST_RD6:REELDRUM_SFB.MEAS 0.745045 0.240001
32DRVMST_RD:COUCH_SFB.MEAS 0.747653 0.238363
28SI0679:28SI0679.PNT \
22AUTOMAX_RD:REELDRUM_SFB.PNT 0.054864
29SI072A:29SA072A.PNT 0.097129
29SI0112:29SA0112.PNT 0.110158
28SI0679:28SI0679.PNT 1.000000
32DRVMST_RD6:REELDRUM_SFB.MEAS 0.301940
32DRVMST_RD:COUCH_SFB.MEAS 0.300307
32DRVMST_RD6:REELDRUM_SFB.MEAS \
22AUTOMAX_RD:REELDRUM_SFB.PNT 0.264184
29SI072A:29SA072A.PNT 0.745045
29SI0112:29SA0112.PNT 0.240001
28SI0679:28SI0679.PNT 0.301940
32DRVMST_RD6:REELDRUM_SFB.MEAS 1.000000
32DRVMST_RD:COUCH_SFB.MEAS 0.997700
32DRVMST_RD:COUCH_SFB.MEAS
22AUTOMAX_RD:REELDRUM_SFB.PNT 0.264724
29SI072A:29SA072A.PNT 0.747653
29SI0112:29SA0112.PNT 0.238363
28SI0679:28SI0679.PNT 0.300307
32DRVMST_RD6:REELDRUM_SFB.MEAS 0.997700
32DRVMST_RD:COUCH_SFB.MEAS 1.000000
Filled missing values in 22AUTOMAX_RD:REELDRUM_SFB.PNT using Random Forest.
Filled missing values in 28SI0679:28SI0679.PNT using Random Forest.
Filled missing values in medium correlation columns using MICE: ['22AUTOMAX_RD:REELDRUM_SFB.PNT', '29SI072A:29SA072A.PNT', '29SI0112:29SA0112.PNT', '28SI0679:28SI0679.PNT', '32DRVMST_RD6:REELDRUM_SFB.MEAS', '32DRVMST_RD:COUCH_SFB.MEAS']
Filled missing values in 22AUTOMAX_RD:REELDRUM_SFB.PNT using forward and backward fill.
Filled missing values in 28SI0679:28SI0679.PNT using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Tank Level:
18LI0028:18LA0028.PNT 19LI0472:19LI0472.PNT \
18LI0028:18LA0028.PNT 1.000000 0.010970
19LI0472:19LI0472.PNT 0.010970 1.000000
19LI0477:19LA0477.PNT -0.035550 -0.131714
22AUTOMAX_RD:PERCENT_DRAW.RO01 0.135941 -0.001824
22AUTOMAX_RD:PERCENT_DRAW.RO02 0.161413 0.012533
... ... ...
28LC0010:28LC0010.MEAS 0.040475 0.068230
28LI0634:28LI0634.PNT 0.140581 -0.025222
28LC0043:28LC0043.MEAS 0.045283 0.101165
32DRVMST_CAL:PERCENT_DRAW.RO02 0.049785 -0.055696
32DRVMST_CAL:PERCENT_DRAW.RO01 -0.004748 0.001862
19LI0477:19LA0477.PNT \
18LI0028:18LA0028.PNT -0.035550
19LI0472:19LI0472.PNT -0.131714
19LI0477:19LA0477.PNT 1.000000
22AUTOMAX_RD:PERCENT_DRAW.RO01 0.039799
22AUTOMAX_RD:PERCENT_DRAW.RO02 0.017086
... ...
28LC0010:28LC0010.MEAS -0.378678
28LI0634:28LI0634.PNT -0.131992
28LC0043:28LC0043.MEAS -0.426312
32DRVMST_CAL:PERCENT_DRAW.RO02 0.148631
32DRVMST_CAL:PERCENT_DRAW.RO01 0.108791
22AUTOMAX_RD:PERCENT_DRAW.RO01 \
18LI0028:18LA0028.PNT 0.135941
19LI0472:19LI0472.PNT -0.001824
19LI0477:19LA0477.PNT 0.039799
22AUTOMAX_RD:PERCENT_DRAW.RO01 1.000000
22AUTOMAX_RD:PERCENT_DRAW.RO02 0.380825
... ...
28LC0010:28LC0010.MEAS 0.022165
28LI0634:28LI0634.PNT -0.012787
28LC0043:28LC0043.MEAS -0.004664
32DRVMST_CAL:PERCENT_DRAW.RO02 0.007659
32DRVMST_CAL:PERCENT_DRAW.RO01 0.063382
22AUTOMAX_RD:PERCENT_DRAW.RO02 \
18LI0028:18LA0028.PNT 0.161413
19LI0472:19LI0472.PNT 0.012533
19LI0477:19LA0477.PNT 0.017086
22AUTOMAX_RD:PERCENT_DRAW.RO01 0.380825
22AUTOMAX_RD:PERCENT_DRAW.RO02 1.000000
... ...
28LC0010:28LC0010.MEAS 0.029880
28LI0634:28LI0634.PNT -0.001440
28LC0043:28LC0043.MEAS 0.026747
32DRVMST_CAL:PERCENT_DRAW.RO02 0.032432
32DRVMST_CAL:PERCENT_DRAW.RO01 0.032809
27LI0475:27LA0475.PNT 28LC1128:28LA1128.MEAS \
18LI0028:18LA0028.PNT 0.248892 -0.040746
19LI0472:19LI0472.PNT 0.036751 -0.057947
19LI0477:19LA0477.PNT -0.026203 0.103958
22AUTOMAX_RD:PERCENT_DRAW.RO01 0.026247 0.004530
22AUTOMAX_RD:PERCENT_DRAW.RO02 0.054477 0.015729
... ... ...
28LC0010:28LC0010.MEAS -0.009800 -0.299405
28LI0634:28LI0634.PNT 0.342430 0.233136
28LC0043:28LC0043.MEAS -0.046990 -0.458985
32DRVMST_CAL:PERCENT_DRAW.RO02 -0.013846 -0.221431
32DRVMST_CAL:PERCENT_DRAW.RO01 -0.000280 -0.037487
28LI0756:28LI0756.PNT 29FC0018:29FE0018.PNT \
18LI0028:18LA0028.PNT -0.068702 0.141417
19LI0472:19LI0472.PNT 0.143073 -0.025241
19LI0477:19LA0477.PNT -0.178693 -0.131912
22AUTOMAX_RD:PERCENT_DRAW.RO01 -0.014359 -0.012676
22AUTOMAX_RD:PERCENT_DRAW.RO02 0.023351 -0.001691
... ... ...
28LC0010:28LC0010.MEAS 0.099954 0.037762
28LI0634:28LI0634.PNT -0.012349 0.999963
28LC0043:28LC0043.MEAS 0.243150 -0.012907
32DRVMST_CAL:PERCENT_DRAW.RO02 0.037439 -0.024469
32DRVMST_CAL:PERCENT_DRAW.RO01 0.015907 0.005603
29FC0074:29FC0074.MEAS ... \
18LI0028:18LA0028.PNT 0.038536 ...
19LI0472:19LI0472.PNT -0.042849 ...
19LI0477:19LA0477.PNT 0.253500 ...
22AUTOMAX_RD:PERCENT_DRAW.RO01 0.048263 ...
22AUTOMAX_RD:PERCENT_DRAW.RO02 -0.023591 ...
... ... ...
28LC0010:28LC0010.MEAS 0.078389 ...
28LI0634:28LI0634.PNT -0.209902 ...
28LC0043:28LC0043.MEAS -0.045802 ...
32DRVMST_CAL:PERCENT_DRAW.RO02 0.043802 ...
32DRVMST_CAL:PERCENT_DRAW.RO01 0.001521 ...
18NC0807:18NC0807.MEAS \
18LI0028:18LA0028.PNT -0.126091
19LI0472:19LI0472.PNT 0.078179
19LI0477:19LA0477.PNT -0.223887
22AUTOMAX_RD:PERCENT_DRAW.RO01 -0.042954
22AUTOMAX_RD:PERCENT_DRAW.RO02 -0.079944
... ...
28LC0010:28LC0010.MEAS 0.251186
28LI0634:28LI0634.PNT -0.006549
28LC0043:28LC0043.MEAS 0.430314
32DRVMST_CAL:PERCENT_DRAW.RO02 0.091292
32DRVMST_CAL:PERCENT_DRAW.RO01 0.029882
18FC0873:18FC0873.MEAS 18LC0806:18LA0806.PNT \
18LI0028:18LA0028.PNT -0.364093 0.069202
19LI0472:19LI0472.PNT 0.019127 0.037169
19LI0477:19LA0477.PNT 0.067310 0.021024
22AUTOMAX_RD:PERCENT_DRAW.RO01 -0.174582 -0.007882
22AUTOMAX_RD:PERCENT_DRAW.RO02 -0.194158 0.005836
... ... ...
28LC0010:28LC0010.MEAS 0.029179 0.134597
28LI0634:28LI0634.PNT -0.003638 0.111619
28LC0043:28LC0043.MEAS 0.003784 0.086940
32DRVMST_CAL:PERCENT_DRAW.RO02 -0.016854 0.103463
32DRVMST_CAL:PERCENT_DRAW.RO01 0.014836 0.021633
18LC0284:18LA0284.PNT 28NC0926:28NC0926.MEAS \
18LI0028:18LA0028.PNT -0.043531 0.074303
19LI0472:19LI0472.PNT 0.053885 -0.060425
19LI0477:19LA0477.PNT 0.018325 0.141969
22AUTOMAX_RD:PERCENT_DRAW.RO01 -0.126128 -0.023817
22AUTOMAX_RD:PERCENT_DRAW.RO02 -0.040154 -0.004089
... ... ...
28LC0010:28LC0010.MEAS 0.086643 -0.129995
28LI0634:28LI0634.PNT 0.173337 0.064946
28LC0043:28LC0043.MEAS -0.015215 -0.272386
32DRVMST_CAL:PERCENT_DRAW.RO02 -0.001974 0.008834
32DRVMST_CAL:PERCENT_DRAW.RO01 0.009518 0.021926
28LC0010:28LC0010.MEAS 28LI0634:28LI0634.PNT \
18LI0028:18LA0028.PNT 0.040475 0.140581
19LI0472:19LI0472.PNT 0.068230 -0.025222
19LI0477:19LA0477.PNT -0.378678 -0.131992
22AUTOMAX_RD:PERCENT_DRAW.RO01 0.022165 -0.012787
22AUTOMAX_RD:PERCENT_DRAW.RO02 0.029880 -0.001440
... ... ...
28LC0010:28LC0010.MEAS 1.000000 0.038374
28LI0634:28LI0634.PNT 0.038374 1.000000
28LC0043:28LC0043.MEAS 0.606646 -0.013289
32DRVMST_CAL:PERCENT_DRAW.RO02 0.261393 -0.023502
32DRVMST_CAL:PERCENT_DRAW.RO01 0.031433 0.005907
28LC0043:28LC0043.MEAS \
18LI0028:18LA0028.PNT 0.045283
19LI0472:19LI0472.PNT 0.101165
19LI0477:19LA0477.PNT -0.426312
22AUTOMAX_RD:PERCENT_DRAW.RO01 -0.004664
22AUTOMAX_RD:PERCENT_DRAW.RO02 0.026747
... ...
28LC0010:28LC0010.MEAS 0.606646
28LI0634:28LI0634.PNT -0.013289
28LC0043:28LC0043.MEAS 1.000000
32DRVMST_CAL:PERCENT_DRAW.RO02 0.245422
32DRVMST_CAL:PERCENT_DRAW.RO01 0.030639
32DRVMST_CAL:PERCENT_DRAW.RO02 \
18LI0028:18LA0028.PNT 0.049785
19LI0472:19LI0472.PNT -0.055696
19LI0477:19LA0477.PNT 0.148631
22AUTOMAX_RD:PERCENT_DRAW.RO01 0.007659
22AUTOMAX_RD:PERCENT_DRAW.RO02 0.032432
... ...
28LC0010:28LC0010.MEAS 0.261393
28LI0634:28LI0634.PNT -0.023502
28LC0043:28LC0043.MEAS 0.245422
32DRVMST_CAL:PERCENT_DRAW.RO02 1.000000
32DRVMST_CAL:PERCENT_DRAW.RO01 -0.024277
32DRVMST_CAL:PERCENT_DRAW.RO01
18LI0028:18LA0028.PNT -0.004748
19LI0472:19LI0472.PNT 0.001862
19LI0477:19LA0477.PNT 0.108791
22AUTOMAX_RD:PERCENT_DRAW.RO01 0.063382
22AUTOMAX_RD:PERCENT_DRAW.RO02 0.032809
... ...
28LC0010:28LC0010.MEAS 0.031433
28LI0634:28LI0634.PNT 0.005907
28LC0043:28LC0043.MEAS 0.030639
32DRVMST_CAL:PERCENT_DRAW.RO02 -0.024277
32DRVMST_CAL:PERCENT_DRAW.RO01 1.000000
[90 rows x 90 columns]
Filled missing values in 18LI0028:18LA0028.PNT using Random Forest.
Filled missing values in 19LI0472:19LI0472.PNT using Random Forest.
Filled missing values in 19LI0477:19LA0477.PNT using Random Forest.
Filled missing values in 27LI0475:27LA0475.PNT using Random Forest.
Filled missing values in 28LC1128:28LA1128.MEAS using Random Forest.
Filled missing values in 28LI0756:28LI0756.PNT using Random Forest.
Filled missing values in 29FC0074:29FC0074.MEAS using Random Forest.
Filled missing values in 29JC0023:29JA0023.PNT using Random Forest.
Filled missing values in 29JC0024:29JA0024.PNT using Random Forest.
Filled missing values in 29LC0113:29LC0113.MEAS using Random Forest.
Filled missing values in 29LC0113:29LC0113.OUT using Random Forest.
Filled missing values in 29LC0134:29LC0134.MEAS using Random Forest.
Filled missing values in 29LC0149:29LCB149.MEAS using Random Forest.
Filled missing values in 32G560:560LT1.PNT using Random Forest.
Filled missing values in 32LC0061:32LC0061.MEAS using Random Forest.
Filled missing values in 32LC0942:32LA0942.MEAS using Random Forest.
Filled missing values in 32LC0951:32LA0951.MEAS using Random Forest.
Filled missing values in 32LC0980:32LA0980.MEAS using Random Forest.
Filled missing values in 32LC1009:32LA1009.MEAS using Random Forest.
Filled missing values in 32LI0426:32LA0426.PNT using Random Forest.
Filled missing values in 32NA1008:32NA1008.MEAS using Random Forest.
Filled missing values in 38FC0654:38FC0654.MEAS using Random Forest.
Filled missing values in 44LI0002:44LI0002.PNT using Random Forest.
Filled missing values in ASA_SIZE:DAY_TK_LVL.PNT using Random Forest.
Filled missing values in ASA_SIZE:SIZE_DAY_TK.PNT using Random Forest.
Filled missing values in 32LI15AA:32LA16AA.PNT using Random Forest.
Filled missing values in 32IT0002:32IT3437A.MEAS using Random Forest.
Filled missing values in BRWN_DYE_LVL:BRWN_DYE_LVL.PNT using Random Forest.
Filled missing values in 29LC0134:29LC0134.MEAS.12hrmax using Random Forest.
Filled missing values in 32PC0140:32PC0140.MEAS using Random Forest.
Filled missing values in 32FC0669:32FB0669.OUT using Random Forest.
Filled missing values in 32FC0677:32FB0677.OUT using Random Forest.
Filled missing values in 32LC0826:32LC0826.OUT using Random Forest.
Filled missing values in 28LC0678:28LC0678.MEAS using Random Forest.
Filled missing values in 18NC0870:18NC0870.MEAS using Random Forest.
Filled missing values in 18NC0807:18NC0807.MEAS using Random Forest.
Filled missing values in 18FC0873:18FC0873.MEAS using Random Forest.
Filled missing values in 18LC0806:18LA0806.PNT using Random Forest.
Filled missing values in 18LC0284:18LA0284.PNT using Random Forest.
Filled missing values in 28NC0926:28NC0926.MEAS using Random Forest.
Filled missing values in 28LC0010:28LC0010.MEAS using Random Forest.
Filled missing values in 28LI0634:28LI0634.PNT using Random Forest.
Filled missing values in 28LC0043:28LC0043.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['18LI0028:18LA0028.PNT', '19LI0472:19LI0472.PNT', '19LI0477:19LA0477.PNT', '22AUTOMAX_RD:PERCENT_DRAW.RO01', '22AUTOMAX_RD:PERCENT_DRAW.RO02', '27LI0475:27LA0475.PNT', '28LC1128:28LA1128.MEAS', '28LI0756:28LI0756.PNT', '29FC0018:29FE0018.PNT', '29FC0074:29FC0074.MEAS', '29FC0114:29FC0114.OUT', '29JC0023:29JA0023.PNT', '29JC0024:29JA0024.PNT', '29LC0030:29LC0030.MEAS', '29LC0071:29LC0071.MEAS', '29LC0088:29LC0088.MEAS', '29LC0106:29LC0106.MEAS', '29LC0113:29LC0113.MEAS', '29LC0113:29LC0113.OUT', '29LC0120:29LC0120.MEAS', '29LC0120:29LC0120.OUT', '29LC0133:29LC0133.MEAS', '29LC0134:29LC0134.MEAS', '29LC0149:29LCB149.MEAS', '29LI0053:29LA0053.PNT', '29LI0174:29LA0174.PNT', '29LI0188:29LA0188.PNT', '29NC0007:29NC0007.OUT', '29NC0076:29NC0076.MEAS', '29NC0076:29NC0076.OUT', '29NC0090:29NB0090.OUT', '29NC0090:29NC0090.MEAS', '32G550:550LT1.PNT', '32G560:560LT1.PNT', '32LC0061:32LC0061.MEAS', '32LC0100:32LC0100.MEAS', '32LC0301:32LC0301.MEAS', '32LC0942:32LA0942.MEAS', '32LC0951:32LA0951.MEAS', '32LC0980:32LA0980.MEAS', '32LC1009:32LA1009.MEAS', '32LI0082:32LA0082.PNT', '32LI0426:32LA0426.PNT', '32NA1008:32NA1008.MEAS', '38FC0654:38FC0654.MEAS', '44LI0002:44LI0002.PNT', 'ASA_SIZE:DAY_TK_LVL.PNT', 'ASA_SIZE:SIZE_DAY_TK.PNT', '32G550:550LT1A.PNT', '32LC0159:32LC0159.MEAS.12hrmax', '32LI15AA:32LA16AA.PNT', '32LC0619:32LC0619.MEAS', '32LC0619:32LC0619.OUT', '29FC0018:29FE0018.PNT.12hrmax', '29LC0088:29LC0088.MEAS.12hrmin', '29LC0230:29LC0230.MEAS.12hrmax', '29LC0230:29LC0230.MEAS.12hrmin', '32IT0002:32IT3437A.MEAS', 'BRWN_DYE_LVL:BRWN_DYE_LVL.PNT', '29LC0134:29LC0134.MEAS.12hrmax', '29LC0149:29LA0149.PNT.12hrmax', '29LC0149:29LA0149.PNT.DailyAvg', '32PC0140:32PC0140.MEAS', '32PC0140:32PC0140.OUT', '32LC0673:32LC0673.MEAS', '32LC0673:32LC0673.OUT', '32FC0669:32FB0669.OUT', '32FC0669:32FC0669.OUT', '32LC0682:32LC0682.MEAS', '32LC0100:32LC0100.OUT.DailyAvg', '32LC0628:32LC0628.OUT', '32FC0677:32FC0677.OUT', '29LC0030:29LC0030.MEAS.12hrmin', '29LC0088:29LC0088.OUT', '32LC0061:32LC0061.OUT', '32FC0677:32FB0677.OUT', '32LC0826:32LC0826.OUT', '32LC0826:32LB0826.OUT', '28LC0678:28LC0678.MEAS', '18NC0870:18NC0870.MEAS', '18NC0807:18NC0807.MEAS', '18FC0873:18FC0873.MEAS', '18LC0806:18LA0806.PNT', '18LC0284:18LA0284.PNT', '28NC0926:28NC0926.MEAS', '28LC0010:28LC0010.MEAS', '28LI0634:28LI0634.PNT', '28LC0043:28LC0043.MEAS', '32DRVMST_CAL:PERCENT_DRAW.RO02', '32DRVMST_CAL:PERCENT_DRAW.RO01']
Filled missing values in 18LI0028:18LA0028.PNT using forward and backward fill.
Filled missing values in 19LI0472:19LI0472.PNT using forward and backward fill.
Filled missing values in 19LI0477:19LA0477.PNT using forward and backward fill.
Filled missing values in 27LI0475:27LA0475.PNT using forward and backward fill.
Filled missing values in 28LC1128:28LA1128.MEAS using forward and backward fill.
Filled missing values in 28LI0756:28LI0756.PNT using forward and backward fill.
Filled missing values in 29FC0074:29FC0074.MEAS using forward and backward fill.
Filled missing values in 29JC0023:29JA0023.PNT using forward and backward fill.
Filled missing values in 29JC0024:29JA0024.PNT using forward and backward fill.
Filled missing values in 29LC0113:29LC0113.MEAS using forward and backward fill.
Filled missing values in 29LC0113:29LC0113.OUT using forward and backward fill.
Filled missing values in 29LC0134:29LC0134.MEAS using forward and backward fill.
Filled missing values in 29LC0149:29LCB149.MEAS using forward and backward fill.
Filled missing values in 32G560:560LT1.PNT using forward and backward fill.
Filled missing values in 32LC0061:32LC0061.MEAS using forward and backward fill.
Filled missing values in 32LC0942:32LA0942.MEAS using forward and backward fill.
Filled missing values in 32LC0951:32LA0951.MEAS using forward and backward fill.
Filled missing values in 32LC0980:32LA0980.MEAS using forward and backward fill.
Filled missing values in 32LC1009:32LA1009.MEAS using forward and backward fill.
Filled missing values in 32LI0426:32LA0426.PNT using forward and backward fill.
Filled missing values in 32NA1008:32NA1008.MEAS using forward and backward fill.
Filled missing values in 38FC0654:38FC0654.MEAS using forward and backward fill.
Filled missing values in 44LI0002:44LI0002.PNT using forward and backward fill.
Filled missing values in ASA_SIZE:DAY_TK_LVL.PNT using forward and backward fill.
Filled missing values in ASA_SIZE:SIZE_DAY_TK.PNT using forward and backward fill.
Filled missing values in 32LI15AA:32LA16AA.PNT using forward and backward fill.
Filled missing values in 32IT0002:32IT3437A.MEAS using forward and backward fill.
Filled missing values in BRWN_DYE_LVL:BRWN_DYE_LVL.PNT using forward and backward fill.
Filled missing values in 29LC0134:29LC0134.MEAS.12hrmax using forward and backward fill.
Filled missing values in 32PC0140:32PC0140.MEAS using forward and backward fill.
Filled missing values in 32FC0669:32FB0669.OUT using forward and backward fill.
Filled missing values in 32FC0677:32FB0677.OUT using forward and backward fill.
Filled missing values in 32LC0826:32LC0826.OUT using forward and backward fill.
Filled missing values in 28LC0678:28LC0678.MEAS using forward and backward fill.
Filled missing values in 18NC0870:18NC0870.MEAS using forward and backward fill.
Filled missing values in 18NC0807:18NC0807.MEAS using forward and backward fill.
Filled missing values in 18FC0873:18FC0873.MEAS using forward and backward fill.
Filled missing values in 18LC0806:18LA0806.PNT using forward and backward fill.
Filled missing values in 18LC0284:18LA0284.PNT using forward and backward fill.
Filled missing values in 28NC0926:28NC0926.MEAS using forward and backward fill.
Filled missing values in 28LC0010:28LC0010.MEAS using forward and backward fill.
Filled missing values in 28LI0634:28LI0634.PNT using forward and backward fill.
Filled missing values in 28LC0043:28LC0043.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Water:
32LC0091:32LC0091A.MEAS 32TI0143:32TA0143.PNT \
32LC0091:32LC0091A.MEAS 1.000000 -0.033542
32TI0143:32TA0143.PNT -0.033542 1.000000
29FC0079:29FC0079.OUT -0.025679 0.089008
32II0486A:32II0486A.PNT 0.026958 0.021407
32FC0304:32FC0304.MEAS -0.064423 0.082124
29FC0079:29FC0079.OUT 32II0486A:32II0486A.PNT \
32LC0091:32LC0091A.MEAS -0.025679 0.026958
32TI0143:32TA0143.PNT 0.089008 0.021407
29FC0079:29FC0079.OUT 1.000000 -0.144573
32II0486A:32II0486A.PNT -0.144573 1.000000
32FC0304:32FC0304.MEAS 0.062171 -0.079847
32FC0304:32FC0304.MEAS
32LC0091:32LC0091A.MEAS -0.064423
32TI0143:32TA0143.PNT 0.082124
29FC0079:29FC0079.OUT 0.062171
32II0486A:32II0486A.PNT -0.079847
32FC0304:32FC0304.MEAS 1.000000
Filled missing values in 32LC0091:32LC0091A.MEAS using Random Forest.
Filled missing values in 32TI0143:32TA0143.PNT using Random Forest.
Filled missing values in 29FC0079:29FC0079.OUT using Random Forest.
Filled missing values in 32II0486A:32II0486A.PNT using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32LC0091:32LC0091A.MEAS', '32TI0143:32TA0143.PNT', '29FC0079:29FC0079.OUT', '32II0486A:32II0486A.PNT', '32FC0304:32FC0304.MEAS']
Filled missing values in 32LC0091:32LC0091A.MEAS using forward and backward fill.
Filled missing values in 32TI0143:32TA0143.PNT using forward and backward fill.
Filled missing values in 29FC0079:29FC0079.OUT using forward and backward fill.
Filled missing values in 32II0486A:32II0486A.PNT using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Dryer:
22AUTOMAX_RD:1STDRYER_DFB.PNT \
22AUTOMAX_RD:1STDRYER_DFB.PNT 1.000000
32PI39AP:32PA39AP.PNT 0.002283
32PI39AQ:32PA39AQ.PNT -0.006741
32PI39AR:32PA39AR.PNT 0.014583
32PI39AS:32PA39AS.PNT 0.046391
32PI39AU:32PA39AU.PNT 0.022608
32PI39AV:32PA39AV.PNT 0.028232
32PI39AW:32PA39AW.PNT 0.039868
32PI39AX:32PA39AX.PNT 0.006166
32PI39AY:32PA39AY.PNT 0.000160
32PI39AZ:32PA39AZ.PNT -0.028484
32PI0933:32PA0933.PNT 0.020889
32PI0934:32PA0934.PNT 0.000665
32PI31AY:32PA31AY.PNT 0.010782
32PI31BI:32PA31BI.PNT 0.044224
32DC0670:32DC0670.OUT 0.018409
32PI0935:32PA0935.PNT 0.015740
32DRVMST_RD3:DRYRCYL1_CUR.MEAS 0.044375
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.050078
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.046377
32DRVMST_RD3:2NDDRYER_CUR.MEAS -0.002148
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.022365
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.009056
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.086494
32DRVMST_RD4:4THDRYER_CUR.MEAS -0.001241
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.078135
32DRVMST_RD4:5THDRYER_CUR.MEAS 0.000455
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.029483
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.046359
32DRVMST_RD4:6THDRYER_DRW.MEAS 0.021551
32DRVMST_RD5:7THDRYER_CUR.MEAS 0.033198
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.023119
32PI39AP:32PA39AP.PNT 32PI39AQ:32PA39AQ.PNT \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.002283 -0.006741
32PI39AP:32PA39AP.PNT 1.000000 0.220489
32PI39AQ:32PA39AQ.PNT 0.220489 1.000000
32PI39AR:32PA39AR.PNT -0.413117 0.202663
32PI39AS:32PA39AS.PNT -0.012579 -0.116335
32PI39AU:32PA39AU.PNT 0.263423 0.094627
32PI39AV:32PA39AV.PNT -0.093077 0.369913
32PI39AW:32PA39AW.PNT 0.262469 0.392615
32PI39AX:32PA39AX.PNT 0.139038 0.426038
32PI39AY:32PA39AY.PNT 0.025464 0.352254
32PI39AZ:32PA39AZ.PNT 0.102894 0.430463
32PI0933:32PA0933.PNT -0.031822 0.058844
32PI0934:32PA0934.PNT 0.118758 0.122751
32PI31AY:32PA31AY.PNT 0.166699 -0.080241
32PI31BI:32PA31BI.PNT 0.106058 -0.128747
32DC0670:32DC0670.OUT -0.097204 -0.304140
32PI0935:32PA0935.PNT -0.082460 -0.159365
32DRVMST_RD3:DRYRCYL1_CUR.MEAS -0.249421 -0.287343
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.128180 0.109102
32DRVMST_RD3:1STDRYER_CUR.MEAS -0.092330 -0.256183
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.192644 -0.230154
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.166859 -0.306386
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.172547 0.140932
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.306933 0.070147
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.053500 -0.189180
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.100370 -0.026533
32DRVMST_RD4:5THDRYER_CUR.MEAS -0.142382 0.304119
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.212973 0.063351
32DRVMST_RD4:6THDRYER_CUR.MEAS -0.087752 0.027786
32DRVMST_RD4:6THDRYER_DRW.MEAS 0.002061 -0.399874
32DRVMST_RD5:7THDRYER_CUR.MEAS -0.110204 0.134960
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.062731 0.303845
32PI39AR:32PA39AR.PNT 32PI39AS:32PA39AS.PNT \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.014583 0.046391
32PI39AP:32PA39AP.PNT -0.413117 -0.012579
32PI39AQ:32PA39AQ.PNT 0.202663 -0.116335
32PI39AR:32PA39AR.PNT 1.000000 0.247693
32PI39AS:32PA39AS.PNT 0.247693 1.000000
32PI39AU:32PA39AU.PNT 0.066163 0.356366
32PI39AV:32PA39AV.PNT 0.208084 0.321202
32PI39AW:32PA39AW.PNT 0.092423 0.058897
32PI39AX:32PA39AX.PNT -0.012838 -0.143117
32PI39AY:32PA39AY.PNT -0.102459 -0.010628
32PI39AZ:32PA39AZ.PNT -0.104734 -0.057366
32PI0933:32PA0933.PNT 0.240419 -0.043205
32PI0934:32PA0934.PNT -0.149555 -0.044251
32PI31AY:32PA31AY.PNT 0.053095 0.083109
32PI31BI:32PA31BI.PNT 0.198146 0.270335
32DC0670:32DC0670.OUT -0.045420 -0.102453
32PI0935:32PA0935.PNT 0.086947 0.003482
32DRVMST_RD3:DRYRCYL1_CUR.MEAS 0.177161 0.362759
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.055319 0.039305
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.147700 0.180308
32DRVMST_RD3:2NDDRYER_CUR.MEAS -0.206689 -0.070832
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.000749 0.187177
32DRVMST_RD4:3RDDRYER_CUR.MEAS -0.102005 0.007209
32DRVMST_RD4:3RDDRYER_DRW.MEAS -0.060029 0.018132
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.091280 -0.003970
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.066048 0.070163
32DRVMST_RD4:5THDRYER_CUR.MEAS 0.099847 -0.074162
32DRVMST_RD4:5THDRYER_DRW.MEAS -0.021917 -0.235301
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.079722 -0.237819
32DRVMST_RD4:6THDRYER_DRW.MEAS -0.035440 0.024817
32DRVMST_RD5:7THDRYER_CUR.MEAS 0.139595 0.237289
32DRVMST_RD5:7THDRYER_DRW.MEAS -0.050915 -0.206742
32PI39AU:32PA39AU.PNT 32PI39AV:32PA39AV.PNT \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.022608 0.028232
32PI39AP:32PA39AP.PNT 0.263423 -0.093077
32PI39AQ:32PA39AQ.PNT 0.094627 0.369913
32PI39AR:32PA39AR.PNT 0.066163 0.208084
32PI39AS:32PA39AS.PNT 0.356366 0.321202
32PI39AU:32PA39AU.PNT 1.000000 0.241415
32PI39AV:32PA39AV.PNT 0.241415 1.000000
32PI39AW:32PA39AW.PNT 0.360218 0.441063
32PI39AX:32PA39AX.PNT 0.229340 0.202862
32PI39AY:32PA39AY.PNT -0.179562 -0.060467
32PI39AZ:32PA39AZ.PNT -0.134555 -0.062892
32PI0933:32PA0933.PNT 0.024786 0.213198
32PI0934:32PA0934.PNT -0.001422 0.008086
32PI31AY:32PA31AY.PNT 0.230209 0.369953
32PI31BI:32PA31BI.PNT 0.313616 0.338371
32DC0670:32DC0670.OUT -0.047683 -0.236277
32PI0935:32PA0935.PNT 0.059320 -0.116921
32DRVMST_RD3:DRYRCYL1_CUR.MEAS 0.188203 0.209449
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.144593 0.090326
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.286101 0.050574
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.134095 -0.104994
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.331487 0.167240
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.179706 0.149700
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.195651 0.195743
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.318089 0.103401
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.077338 0.149207
32DRVMST_RD4:5THDRYER_CUR.MEAS -0.116174 0.001603
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.096776 0.037600
32DRVMST_RD4:6THDRYER_CUR.MEAS -0.119479 -0.134884
32DRVMST_RD4:6THDRYER_DRW.MEAS -0.093728 -0.211510
32DRVMST_RD5:7THDRYER_CUR.MEAS -0.068225 0.019112
32DRVMST_RD5:7THDRYER_DRW.MEAS -0.170368 -0.175208
32PI39AW:32PA39AW.PNT 32PI39AX:32PA39AX.PNT \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.039868 0.006166
32PI39AP:32PA39AP.PNT 0.262469 0.139038
32PI39AQ:32PA39AQ.PNT 0.392615 0.426038
32PI39AR:32PA39AR.PNT 0.092423 -0.012838
32PI39AS:32PA39AS.PNT 0.058897 -0.143117
32PI39AU:32PA39AU.PNT 0.360218 0.229340
32PI39AV:32PA39AV.PNT 0.441063 0.202862
32PI39AW:32PA39AW.PNT 1.000000 0.419269
32PI39AX:32PA39AX.PNT 0.419269 1.000000
32PI39AY:32PA39AY.PNT -0.201474 0.099480
32PI39AZ:32PA39AZ.PNT -0.126164 0.151115
32PI0933:32PA0933.PNT 0.434580 0.016431
32PI0934:32PA0934.PNT 0.099102 0.046562
32PI31AY:32PA31AY.PNT 0.316600 -0.213925
32PI31BI:32PA31BI.PNT 0.300613 -0.116068
32DC0670:32DC0670.OUT -0.077830 0.006622
32PI0935:32PA0935.PNT 0.067323 0.035328
32DRVMST_RD3:DRYRCYL1_CUR.MEAS 0.079680 -0.230081
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.167924 0.157630
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.283846 -0.056588
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.030053 -0.143590
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.079082 -0.214921
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.392105 0.472078
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.281957 0.067697
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.350748 0.011759
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.136163 -0.015206
32DRVMST_RD4:5THDRYER_CUR.MEAS -0.080773 0.183889
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.162227 0.008078
32DRVMST_RD4:6THDRYER_CUR.MEAS -0.044372 0.165152
32DRVMST_RD4:6THDRYER_DRW.MEAS -0.388000 -0.515729
32DRVMST_RD5:7THDRYER_CUR.MEAS -0.038766 0.108406
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.002126 0.264192
32PI39AY:32PA39AY.PNT ... \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.000160 ...
32PI39AP:32PA39AP.PNT 0.025464 ...
32PI39AQ:32PA39AQ.PNT 0.352254 ...
32PI39AR:32PA39AR.PNT -0.102459 ...
32PI39AS:32PA39AS.PNT -0.010628 ...
32PI39AU:32PA39AU.PNT -0.179562 ...
32PI39AV:32PA39AV.PNT -0.060467 ...
32PI39AW:32PA39AW.PNT -0.201474 ...
32PI39AX:32PA39AX.PNT 0.099480 ...
32PI39AY:32PA39AY.PNT 1.000000 ...
32PI39AZ:32PA39AZ.PNT 0.759059 ...
32PI0933:32PA0933.PNT -0.445448 ...
32PI0934:32PA0934.PNT 0.219332 ...
32PI31AY:32PA31AY.PNT -0.578243 ...
32PI31BI:32PA31BI.PNT -0.497175 ...
32DC0670:32DC0670.OUT -0.060004 ...
32PI0935:32PA0935.PNT 0.081911 ...
32DRVMST_RD3:DRYRCYL1_CUR.MEAS -0.502567 ...
32DRVMST_RD3:1STDRYER_DRW.MEAS -0.272295 ...
32DRVMST_RD3:1STDRYER_CUR.MEAS -0.499739 ...
32DRVMST_RD3:2NDDRYER_CUR.MEAS -0.350424 ...
32DRVMST_RD3:2NDDRYER_DRW.MEAS -0.526590 ...
32DRVMST_RD4:3RDDRYER_CUR.MEAS -0.009634 ...
32DRVMST_RD4:3RDDRYER_DRW.MEAS -0.306919 ...
32DRVMST_RD4:4THDRYER_CUR.MEAS -0.482811 ...
32DRVMST_RD4:4THDRYER_DRW.MEAS -0.297192 ...
32DRVMST_RD4:5THDRYER_CUR.MEAS 0.188191 ...
32DRVMST_RD4:5THDRYER_DRW.MEAS -0.160937 ...
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.168182 ...
32DRVMST_RD4:6THDRYER_DRW.MEAS -0.197573 ...
32DRVMST_RD5:7THDRYER_CUR.MEAS 0.118826 ...
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.314648 ...
32DRVMST_RD4:3RDDRYER_CUR.MEAS \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.009056
32PI39AP:32PA39AP.PNT 0.172547
32PI39AQ:32PA39AQ.PNT 0.140932
32PI39AR:32PA39AR.PNT -0.102005
32PI39AS:32PA39AS.PNT 0.007209
32PI39AU:32PA39AU.PNT 0.179706
32PI39AV:32PA39AV.PNT 0.149700
32PI39AW:32PA39AW.PNT 0.392105
32PI39AX:32PA39AX.PNT 0.472078
32PI39AY:32PA39AY.PNT -0.009634
32PI39AZ:32PA39AZ.PNT -0.004691
32PI0933:32PA0933.PNT 0.241765
32PI0934:32PA0934.PNT 0.040447
32PI31AY:32PA31AY.PNT -0.137172
32PI31BI:32PA31BI.PNT -0.068504
32DC0670:32DC0670.OUT 0.075089
32PI0935:32PA0935.PNT 0.105595
32DRVMST_RD3:DRYRCYL1_CUR.MEAS -0.075244
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.283934
32DRVMST_RD3:1STDRYER_CUR.MEAS -0.007193
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.140883
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.109545
32DRVMST_RD4:3RDDRYER_CUR.MEAS 1.000000
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.171201
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.219842
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.060733
32DRVMST_RD4:5THDRYER_CUR.MEAS 0.075995
32DRVMST_RD4:5THDRYER_DRW.MEAS -0.025931
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.188841
32DRVMST_RD4:6THDRYER_DRW.MEAS -0.277212
32DRVMST_RD5:7THDRYER_CUR.MEAS 0.195513
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.339540
32DRVMST_RD4:3RDDRYER_DRW.MEAS \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.086494
32PI39AP:32PA39AP.PNT 0.306933
32PI39AQ:32PA39AQ.PNT 0.070147
32PI39AR:32PA39AR.PNT -0.060029
32PI39AS:32PA39AS.PNT 0.018132
32PI39AU:32PA39AU.PNT 0.195651
32PI39AV:32PA39AV.PNT 0.195743
32PI39AW:32PA39AW.PNT 0.281957
32PI39AX:32PA39AX.PNT 0.067697
32PI39AY:32PA39AY.PNT -0.306919
32PI39AZ:32PA39AZ.PNT -0.190787
32PI0933:32PA0933.PNT 0.322855
32PI0934:32PA0934.PNT -0.053112
32PI31AY:32PA31AY.PNT 0.358608
32PI31BI:32PA31BI.PNT 0.278416
32DC0670:32DC0670.OUT -0.143487
32PI0935:32PA0935.PNT 0.004854
32DRVMST_RD3:DRYRCYL1_CUR.MEAS 0.168390
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.644921
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.345340
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.254443
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.583190
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.171201
32DRVMST_RD4:3RDDRYER_DRW.MEAS 1.000000
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.183667
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.661193
32DRVMST_RD4:5THDRYER_CUR.MEAS -0.026743
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.513160
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.082186
32DRVMST_RD4:6THDRYER_DRW.MEAS 0.251082
32DRVMST_RD5:7THDRYER_CUR.MEAS 0.000656
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.172179
32DRVMST_RD4:4THDRYER_CUR.MEAS \
22AUTOMAX_RD:1STDRYER_DFB.PNT -0.001241
32PI39AP:32PA39AP.PNT 0.053500
32PI39AQ:32PA39AQ.PNT -0.189180
32PI39AR:32PA39AR.PNT 0.091280
32PI39AS:32PA39AS.PNT -0.003970
32PI39AU:32PA39AU.PNT 0.318089
32PI39AV:32PA39AV.PNT 0.103401
32PI39AW:32PA39AW.PNT 0.350748
32PI39AX:32PA39AX.PNT 0.011759
32PI39AY:32PA39AY.PNT -0.482811
32PI39AZ:32PA39AZ.PNT -0.562876
32PI0933:32PA0933.PNT 0.365827
32PI0934:32PA0934.PNT 0.003370
32PI31AY:32PA31AY.PNT 0.521807
32PI31BI:32PA31BI.PNT 0.481788
32DC0670:32DC0670.OUT 0.187643
32PI0935:32PA0935.PNT 0.189089
32DRVMST_RD3:DRYRCYL1_CUR.MEAS 0.430254
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.145410
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.529958
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.422070
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.379731
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.219842
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.183667
32DRVMST_RD4:4THDRYER_CUR.MEAS 1.000000
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.218096
32DRVMST_RD4:5THDRYER_CUR.MEAS -0.279908
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.331622
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.074036
32DRVMST_RD4:6THDRYER_DRW.MEAS 0.098848
32DRVMST_RD5:7THDRYER_CUR.MEAS -0.390275
32DRVMST_RD5:7THDRYER_DRW.MEAS -0.293623
32DRVMST_RD4:4THDRYER_DRW.MEAS \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.078135
32PI39AP:32PA39AP.PNT 0.100370
32PI39AQ:32PA39AQ.PNT -0.026533
32PI39AR:32PA39AR.PNT 0.066048
32PI39AS:32PA39AS.PNT 0.070163
32PI39AU:32PA39AU.PNT 0.077338
32PI39AV:32PA39AV.PNT 0.149207
32PI39AW:32PA39AW.PNT 0.136163
32PI39AX:32PA39AX.PNT -0.015206
32PI39AY:32PA39AY.PNT -0.297192
32PI39AZ:32PA39AZ.PNT -0.235443
32PI0933:32PA0933.PNT 0.227979
32PI0934:32PA0934.PNT -0.094999
32PI31AY:32PA31AY.PNT 0.267099
32PI31BI:32PA31BI.PNT 0.275490
32DC0670:32DC0670.OUT -0.088611
32PI0935:32PA0935.PNT -0.058319
32DRVMST_RD3:DRYRCYL1_CUR.MEAS 0.275058
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.652473
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.297285
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.323174
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.436461
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.060733
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.661193
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.218096
32DRVMST_RD4:4THDRYER_DRW.MEAS 1.000000
32DRVMST_RD4:5THDRYER_CUR.MEAS 0.139672
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.405544
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.207512
32DRVMST_RD4:6THDRYER_DRW.MEAS 0.406846
32DRVMST_RD5:7THDRYER_CUR.MEAS 0.133918
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.223022
32DRVMST_RD4:5THDRYER_CUR.MEAS \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.000455
32PI39AP:32PA39AP.PNT -0.142382
32PI39AQ:32PA39AQ.PNT 0.304119
32PI39AR:32PA39AR.PNT 0.099847
32PI39AS:32PA39AS.PNT -0.074162
32PI39AU:32PA39AU.PNT -0.116174
32PI39AV:32PA39AV.PNT 0.001603
32PI39AW:32PA39AW.PNT -0.080773
32PI39AX:32PA39AX.PNT 0.183889
32PI39AY:32PA39AY.PNT 0.188191
32PI39AZ:32PA39AZ.PNT 0.291261
32PI0933:32PA0933.PNT -0.146095
32PI0934:32PA0934.PNT -0.139119
32PI31AY:32PA31AY.PNT -0.323479
32PI31BI:32PA31BI.PNT -0.279056
32DC0670:32DC0670.OUT -0.127260
32PI0935:32PA0935.PNT -0.350394
32DRVMST_RD3:DRYRCYL1_CUR.MEAS -0.122289
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.310907
32DRVMST_RD3:1STDRYER_CUR.MEAS -0.316540
32DRVMST_RD3:2NDDRYER_CUR.MEAS -0.132892
32DRVMST_RD3:2NDDRYER_DRW.MEAS -0.284925
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.075995
32DRVMST_RD4:3RDDRYER_DRW.MEAS -0.026743
32DRVMST_RD4:4THDRYER_CUR.MEAS -0.279908
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.139672
32DRVMST_RD4:5THDRYER_CUR.MEAS 1.000000
32DRVMST_RD4:5THDRYER_DRW.MEAS -0.156415
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.079050
32DRVMST_RD4:6THDRYER_DRW.MEAS -0.051261
32DRVMST_RD5:7THDRYER_CUR.MEAS 0.448892
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.410038
32DRVMST_RD4:5THDRYER_DRW.MEAS \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.029483
32PI39AP:32PA39AP.PNT 0.212973
32PI39AQ:32PA39AQ.PNT 0.063351
32PI39AR:32PA39AR.PNT -0.021917
32PI39AS:32PA39AS.PNT -0.235301
32PI39AU:32PA39AU.PNT 0.096776
32PI39AV:32PA39AV.PNT 0.037600
32PI39AW:32PA39AW.PNT 0.162227
32PI39AX:32PA39AX.PNT 0.008078
32PI39AY:32PA39AY.PNT -0.160937
32PI39AZ:32PA39AZ.PNT -0.182342
32PI0933:32PA0933.PNT 0.308773
32PI0934:32PA0934.PNT 0.135358
32PI31AY:32PA31AY.PNT 0.436731
32PI31BI:32PA31BI.PNT 0.320648
32DC0670:32DC0670.OUT -0.054092
32PI0935:32PA0935.PNT 0.352724
32DRVMST_RD3:DRYRCYL1_CUR.MEAS 0.125417
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.354298
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.313221
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.352428
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.408090
32DRVMST_RD4:3RDDRYER_CUR.MEAS -0.025931
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.513160
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.331622
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.405544
32DRVMST_RD4:5THDRYER_CUR.MEAS -0.156415
32DRVMST_RD4:5THDRYER_DRW.MEAS 1.000000
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.309113
32DRVMST_RD4:6THDRYER_DRW.MEAS 0.355495
32DRVMST_RD5:7THDRYER_CUR.MEAS -0.434696
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.094392
32DRVMST_RD4:6THDRYER_CUR.MEAS \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.046359
32PI39AP:32PA39AP.PNT -0.087752
32PI39AQ:32PA39AQ.PNT 0.027786
32PI39AR:32PA39AR.PNT 0.079722
32PI39AS:32PA39AS.PNT -0.237819
32PI39AU:32PA39AU.PNT -0.119479
32PI39AV:32PA39AV.PNT -0.134884
32PI39AW:32PA39AW.PNT -0.044372
32PI39AX:32PA39AX.PNT 0.165152
32PI39AY:32PA39AY.PNT 0.168182
32PI39AZ:32PA39AZ.PNT -0.024778
32PI0933:32PA0933.PNT -0.048532
32PI0934:32PA0934.PNT 0.086915
32PI31AY:32PA31AY.PNT -0.152546
32PI31BI:32PA31BI.PNT -0.017123
32DC0670:32DC0670.OUT -0.051535
32PI0935:32PA0935.PNT 0.285185
32DRVMST_RD3:DRYRCYL1_CUR.MEAS -0.087594
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.235850
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.002296
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.136976
32DRVMST_RD3:2NDDRYER_DRW.MEAS -0.037683
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.188841
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.082186
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.074036
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.207512
32DRVMST_RD4:5THDRYER_CUR.MEAS 0.079050
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.309113
32DRVMST_RD4:6THDRYER_CUR.MEAS 1.000000
32DRVMST_RD4:6THDRYER_DRW.MEAS 0.118095
32DRVMST_RD5:7THDRYER_CUR.MEAS -0.074969
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.343869
32DRVMST_RD4:6THDRYER_DRW.MEAS \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.021551
32PI39AP:32PA39AP.PNT 0.002061
32PI39AQ:32PA39AQ.PNT -0.399874
32PI39AR:32PA39AR.PNT -0.035440
32PI39AS:32PA39AS.PNT 0.024817
32PI39AU:32PA39AU.PNT -0.093728
32PI39AV:32PA39AV.PNT -0.211510
32PI39AW:32PA39AW.PNT -0.388000
32PI39AX:32PA39AX.PNT -0.515729
32PI39AY:32PA39AY.PNT -0.197573
32PI39AZ:32PA39AZ.PNT -0.166037
32PI0933:32PA0933.PNT 0.130176
32PI0934:32PA0934.PNT -0.049085
32PI31AY:32PA31AY.PNT 0.206550
32PI31BI:32PA31BI.PNT 0.082486
32DC0670:32DC0670.OUT -0.086316
32PI0935:32PA0935.PNT 0.013221
32DRVMST_RD3:DRYRCYL1_CUR.MEAS 0.223332
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.245555
32DRVMST_RD3:1STDRYER_CUR.MEAS 0.142480
32DRVMST_RD3:2NDDRYER_CUR.MEAS 0.299259
32DRVMST_RD3:2NDDRYER_DRW.MEAS 0.421691
32DRVMST_RD4:3RDDRYER_CUR.MEAS -0.277212
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.251082
32DRVMST_RD4:4THDRYER_CUR.MEAS 0.098848
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.406846
32DRVMST_RD4:5THDRYER_CUR.MEAS -0.051261
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.355495
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.118095
32DRVMST_RD4:6THDRYER_DRW.MEAS 1.000000
32DRVMST_RD5:7THDRYER_CUR.MEAS -0.098975
32DRVMST_RD5:7THDRYER_DRW.MEAS -0.031715
32DRVMST_RD5:7THDRYER_CUR.MEAS \
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.033198
32PI39AP:32PA39AP.PNT -0.110204
32PI39AQ:32PA39AQ.PNT 0.134960
32PI39AR:32PA39AR.PNT 0.139595
32PI39AS:32PA39AS.PNT 0.237289
32PI39AU:32PA39AU.PNT -0.068225
32PI39AV:32PA39AV.PNT 0.019112
32PI39AW:32PA39AW.PNT -0.038766
32PI39AX:32PA39AX.PNT 0.108406
32PI39AY:32PA39AY.PNT 0.118826
32PI39AZ:32PA39AZ.PNT 0.236221
32PI0933:32PA0933.PNT -0.082273
32PI0934:32PA0934.PNT -0.121335
32PI31AY:32PA31AY.PNT -0.369158
32PI31BI:32PA31BI.PNT -0.213504
32DC0670:32DC0670.OUT -0.016408
32PI0935:32PA0935.PNT -0.315092
32DRVMST_RD3:DRYRCYL1_CUR.MEAS -0.122499
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.265415
32DRVMST_RD3:1STDRYER_CUR.MEAS -0.233176
32DRVMST_RD3:2NDDRYER_CUR.MEAS -0.121931
32DRVMST_RD3:2NDDRYER_DRW.MEAS -0.140998
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.195513
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.000656
32DRVMST_RD4:4THDRYER_CUR.MEAS -0.390275
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.133918
32DRVMST_RD4:5THDRYER_CUR.MEAS 0.448892
32DRVMST_RD4:5THDRYER_DRW.MEAS -0.434696
32DRVMST_RD4:6THDRYER_CUR.MEAS -0.074969
32DRVMST_RD4:6THDRYER_DRW.MEAS -0.098975
32DRVMST_RD5:7THDRYER_CUR.MEAS 1.000000
32DRVMST_RD5:7THDRYER_DRW.MEAS 0.313402
32DRVMST_RD5:7THDRYER_DRW.MEAS
22AUTOMAX_RD:1STDRYER_DFB.PNT 0.023119
32PI39AP:32PA39AP.PNT 0.062731
32PI39AQ:32PA39AQ.PNT 0.303845
32PI39AR:32PA39AR.PNT -0.050915
32PI39AS:32PA39AS.PNT -0.206742
32PI39AU:32PA39AU.PNT -0.170368
32PI39AV:32PA39AV.PNT -0.175208
32PI39AW:32PA39AW.PNT 0.002126
32PI39AX:32PA39AX.PNT 0.264192
32PI39AY:32PA39AY.PNT 0.314648
32PI39AZ:32PA39AZ.PNT 0.434074
32PI0933:32PA0933.PNT -0.004566
32PI0934:32PA0934.PNT 0.050274
32PI31AY:32PA31AY.PNT -0.438853
32PI31BI:32PA31BI.PNT -0.411754
32DC0670:32DC0670.OUT -0.087733
32PI0935:32PA0935.PNT -0.028807
32DRVMST_RD3:DRYRCYL1_CUR.MEAS -0.336476
32DRVMST_RD3:1STDRYER_DRW.MEAS 0.365194
32DRVMST_RD3:1STDRYER_CUR.MEAS -0.270287
32DRVMST_RD3:2NDDRYER_CUR.MEAS -0.074948
32DRVMST_RD3:2NDDRYER_DRW.MEAS -0.156839
32DRVMST_RD4:3RDDRYER_CUR.MEAS 0.339540
32DRVMST_RD4:3RDDRYER_DRW.MEAS 0.172179
32DRVMST_RD4:4THDRYER_CUR.MEAS -0.293623
32DRVMST_RD4:4THDRYER_DRW.MEAS 0.223022
32DRVMST_RD4:5THDRYER_CUR.MEAS 0.410038
32DRVMST_RD4:5THDRYER_DRW.MEAS 0.094392
32DRVMST_RD4:6THDRYER_CUR.MEAS 0.343869
32DRVMST_RD4:6THDRYER_DRW.MEAS -0.031715
32DRVMST_RD5:7THDRYER_CUR.MEAS 0.313402
32DRVMST_RD5:7THDRYER_DRW.MEAS 1.000000
[32 rows x 32 columns]
Filled missing values in 22AUTOMAX_RD:1STDRYER_DFB.PNT using Random Forest.
Filled missing values in 32PI39AP:32PA39AP.PNT using Random Forest.
Filled missing values in 32PI39AQ:32PA39AQ.PNT using Random Forest.
Filled missing values in 32PI39AR:32PA39AR.PNT using Random Forest.
Filled missing values in 32PI39AS:32PA39AS.PNT using Random Forest.
Filled missing values in 32PI39AU:32PA39AU.PNT using Random Forest.
Filled missing values in 32PI39AV:32PA39AV.PNT using Random Forest.
Filled missing values in 32PI39AW:32PA39AW.PNT using Random Forest.
Filled missing values in 32PI39AX:32PA39AX.PNT using Random Forest.
Filled missing values in 32PI39AY:32PA39AY.PNT using Random Forest.
Filled missing values in 32PI39AZ:32PA39AZ.PNT using Random Forest.
Filled missing values in 32PI0933:32PA0933.PNT using Random Forest.
Filled missing values in 32PI0934:32PA0934.PNT using Random Forest.
Filled missing values in 32PI0935:32PA0935.PNT using Random Forest.
Filled missing values in 32DRVMST_RD3:1STDRYER_DRW.MEAS using Random Forest.
Filled missing values in 32DRVMST_RD3:2NDDRYER_DRW.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['22AUTOMAX_RD:1STDRYER_DFB.PNT', '32PI39AP:32PA39AP.PNT', '32PI39AQ:32PA39AQ.PNT', '32PI39AR:32PA39AR.PNT', '32PI39AS:32PA39AS.PNT', '32PI39AU:32PA39AU.PNT', '32PI39AV:32PA39AV.PNT', '32PI39AW:32PA39AW.PNT', '32PI39AX:32PA39AX.PNT', '32PI39AY:32PA39AY.PNT', '32PI39AZ:32PA39AZ.PNT', '32PI0933:32PA0933.PNT', '32PI0934:32PA0934.PNT', '32PI31AY:32PA31AY.PNT', '32PI31BI:32PA31BI.PNT', '32DC0670:32DC0670.OUT', '32PI0935:32PA0935.PNT', '32DRVMST_RD3:DRYRCYL1_CUR.MEAS', '32DRVMST_RD3:1STDRYER_DRW.MEAS', '32DRVMST_RD3:1STDRYER_CUR.MEAS', '32DRVMST_RD3:2NDDRYER_CUR.MEAS', '32DRVMST_RD3:2NDDRYER_DRW.MEAS', '32DRVMST_RD4:3RDDRYER_CUR.MEAS', '32DRVMST_RD4:3RDDRYER_DRW.MEAS', '32DRVMST_RD4:4THDRYER_CUR.MEAS', '32DRVMST_RD4:4THDRYER_DRW.MEAS', '32DRVMST_RD4:5THDRYER_CUR.MEAS', '32DRVMST_RD4:5THDRYER_DRW.MEAS', '32DRVMST_RD4:6THDRYER_CUR.MEAS', '32DRVMST_RD4:6THDRYER_DRW.MEAS', '32DRVMST_RD5:7THDRYER_CUR.MEAS', '32DRVMST_RD5:7THDRYER_DRW.MEAS']
Filled missing values in 22AUTOMAX_RD:1STDRYER_DFB.PNT using forward and backward fill.
Filled missing values in 32PI39AP:32PA39AP.PNT using forward and backward fill.
Filled missing values in 32PI39AQ:32PA39AQ.PNT using forward and backward fill.
Filled missing values in 32PI39AR:32PA39AR.PNT using forward and backward fill.
Filled missing values in 32PI39AS:32PA39AS.PNT using forward and backward fill.
Filled missing values in 32PI39AU:32PA39AU.PNT using forward and backward fill.
Filled missing values in 32PI39AV:32PA39AV.PNT using forward and backward fill.
Filled missing values in 32PI39AW:32PA39AW.PNT using forward and backward fill.
Filled missing values in 32PI39AX:32PA39AX.PNT using forward and backward fill.
Filled missing values in 32PI39AY:32PA39AY.PNT using forward and backward fill.
Filled missing values in 32PI39AZ:32PA39AZ.PNT using forward and backward fill.
Filled missing values in 32PI0933:32PA0933.PNT using forward and backward fill.
Filled missing values in 32PI0934:32PA0934.PNT using forward and backward fill.
Filled missing values in 32PI0935:32PA0935.PNT using forward and backward fill.
Filled missing values in 32DRVMST_RD3:1STDRYER_DRW.MEAS using forward and backward fill.
Filled missing values in 32DRVMST_RD3:2NDDRYER_DRW.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Steam:
32FC0121:32FC0121.MEAS 32FI0607:32FA0607.PNT \
32FC0121:32FC0121.MEAS 1.000000 -0.429052
32FI0607:32FA0607.PNT -0.429052 1.000000
32FI0610:32FA0610.PNT 0.488062 -0.263028
32PC0611:32PC0611.MEAS -0.420925 0.723388
32PC0611:32PC0611.OUT 0.224231 -0.149974
32PC0613:32PC0613.MEAS -0.358096 0.735822
32PC0614:32PC0614.MEAS -0.341396 0.710980
32PC0615:32PC0615.MEAS -0.360330 0.735976
32PC0623:32PB0623.OUT -0.437463 0.760814
32PC0623:32PC0623.MEAS -0.406653 0.810911
32PC0635:32PB0635.OUT -0.302877 0.821940
32PC0635:32PC0635.MEAS -0.409903 0.814425
32PC0665:32PB0665.OUT -0.500168 0.811571
32PC0665:32PC0665.MEAS -0.413120 0.817695
32PC0676:32PB0676.OUT -0.344938 0.829166
32PC0676:32PC0676.MEAS -0.417793 0.819903
32PI0609:32PA0609.PNT 0.049651 -0.307989
32PI0625:32PA0625.PNT 0.238506 -0.190032
40PC0060:40PA0060.PNT -0.042230 -0.122931
40TC0074:40TC0074.MEAS 0.094998 0.063075
32PC0616:32PC0616.OUT 0.417411 -0.732867
32PC0676:32PA0676.PNT -0.417782 0.819877
32FI0637:32FA0637.PNT 0.083134 -0.087661
32FI0650:32FA0650.PNT -0.371644 0.840685
32PC0616:32PC0616.MEAS -0.404226 0.807063
32PC0617:32PC0617.MEAS -0.437322 0.685285
32PC0676:32PC0676.OUT -0.343931 0.828213
32FI0648:32FA0648.PNT -0.363981 0.847098
32PC0635:32PC0635.OUT -0.303126 0.822049
32TC0727:32TC0727.MEAS -0.406059 0.359103
PM2SteamKpphTon 0.270986 0.129926
40MAINS:40_0102.PNT_3 0.272100 -0.217564
32FI0610:32FA0610.PNT 32PC0611:32PC0611.MEAS \
32FC0121:32FC0121.MEAS 0.488062 -0.420925
32FI0607:32FA0607.PNT -0.263028 0.723388
32FI0610:32FA0610.PNT 1.000000 -0.305262
32PC0611:32PC0611.MEAS -0.305262 1.000000
32PC0611:32PC0611.OUT -0.080226 -0.098227
32PC0613:32PC0613.MEAS -0.219134 0.895468
32PC0614:32PC0614.MEAS -0.193276 0.825699
32PC0615:32PC0615.MEAS -0.221218 0.895104
32PC0623:32PB0623.OUT -0.262171 0.726499
32PC0623:32PC0623.MEAS -0.190003 0.799441
32PC0635:32PB0635.OUT -0.180287 0.754919
32PC0635:32PC0635.MEAS -0.195341 0.805104
32PC0665:32PB0665.OUT -0.276612 0.756284
32PC0665:32PC0665.MEAS -0.204726 0.810036
32PC0676:32PB0676.OUT -0.217905 0.738157
32PC0676:32PC0676.MEAS -0.217223 0.815515
32PI0609:32PA0609.PNT 0.141283 -0.188786
32PI0625:32PA0625.PNT 0.156449 -0.158784
40PC0060:40PA0060.PNT 0.103946 -0.047293
40TC0074:40TC0074.MEAS 0.302734 0.016222
32PC0616:32PC0616.OUT 0.291228 -0.712342
32PC0676:32PA0676.PNT -0.217213 0.815486
32FI0637:32FA0637.PNT -0.143325 -0.048917
32FI0650:32FA0650.PNT -0.264727 0.767864
32PC0616:32PC0616.MEAS -0.188644 0.798990
32PC0617:32PC0617.MEAS -0.217246 0.716719
32PC0676:32PC0676.OUT -0.216996 0.735180
32FI0648:32FA0648.PNT -0.287463 0.792893
32PC0635:32PC0635.OUT -0.180220 0.754611
32TC0727:32TC0727.MEAS -0.341058 0.620756
PM2SteamKpphTon 0.215561 -0.160520
40MAINS:40_0102.PNT_3 0.293495 -0.203608
32PC0611:32PC0611.OUT 32PC0613:32PC0613.MEAS \
32FC0121:32FC0121.MEAS 0.224231 -0.358096
32FI0607:32FA0607.PNT -0.149974 0.735822
32FI0610:32FA0610.PNT -0.080226 -0.219134
32PC0611:32PC0611.MEAS -0.098227 0.895468
32PC0611:32PC0611.OUT 1.000000 -0.297348
32PC0613:32PC0613.MEAS -0.297348 1.000000
32PC0614:32PC0614.MEAS -0.290370 0.930825
32PC0615:32PC0615.MEAS -0.295511 0.999425
32PC0623:32PB0623.OUT -0.309208 0.752780
32PC0623:32PC0623.MEAS -0.489726 0.884394
32PC0635:32PB0635.OUT -0.288978 0.812714
32PC0635:32PC0635.MEAS -0.483092 0.889187
32PC0665:32PB0665.OUT -0.440027 0.822374
32PC0665:32PC0665.MEAS -0.477805 0.894010
32PC0676:32PB0676.OUT -0.380654 0.803596
32PC0676:32PC0676.MEAS -0.469728 0.899162
32PI0609:32PA0609.PNT -0.203814 -0.189199
32PI0625:32PA0625.PNT 0.106077 -0.168815
40PC0060:40PA0060.PNT -0.251076 -0.047736
40TC0074:40TC0074.MEAS -0.104147 0.039348
32PC0616:32PC0616.OUT 0.370103 -0.805051
32PC0676:32PA0676.PNT -0.469755 0.899162
32FI0637:32FA0637.PNT 0.956610 -0.274318
32FI0650:32FA0650.PNT -0.362271 0.810740
32PC0616:32PC0616.MEAS -0.491503 0.885037
32PC0617:32PC0617.MEAS -0.487945 0.787477
32PC0676:32PC0676.OUT -0.378932 0.800554
32FI0648:32FA0648.PNT -0.232162 0.827481
32PC0635:32PC0635.OUT -0.288992 0.812345
32TC0727:32TC0727.MEAS -0.312901 0.629345
PM2SteamKpphTon 0.254943 -0.168569
40MAINS:40_0102.PNT_3 0.095499 -0.207388
32PC0614:32PC0614.MEAS 32PC0615:32PC0615.MEAS \
32FC0121:32FC0121.MEAS -0.341396 -0.360330
32FI0607:32FA0607.PNT 0.710980 0.735976
32FI0610:32FA0610.PNT -0.193276 -0.221218
32PC0611:32PC0611.MEAS 0.825699 0.895104
32PC0611:32PC0611.OUT -0.290370 -0.295511
32PC0613:32PC0613.MEAS 0.930825 0.999425
32PC0614:32PC0614.MEAS 1.000000 0.931453
32PC0615:32PC0615.MEAS 0.931453 1.000000
32PC0623:32PB0623.OUT 0.625777 0.751950
32PC0623:32PC0623.MEAS 0.810673 0.883806
32PC0635:32PB0635.OUT 0.740411 0.812381
32PC0635:32PC0635.MEAS 0.813737 0.888471
32PC0665:32PB0665.OUT 0.761223 0.822752
32PC0665:32PC0665.MEAS 0.816714 0.893285
32PC0676:32PB0676.OUT 0.756483 0.803410
32PC0676:32PC0676.MEAS 0.822189 0.898376
32PI0609:32PA0609.PNT -0.196288 -0.191580
32PI0625:32PA0625.PNT -0.191209 -0.169511
40PC0060:40PA0060.PNT -0.055920 -0.050214
40TC0074:40TC0074.MEAS 0.025251 0.038913
32PC0616:32PC0616.OUT -0.738614 -0.805859
32PC0676:32PA0676.PNT 0.822222 0.898376
32FI0637:32FA0637.PNT -0.277060 -0.273119
32FI0650:32FA0650.PNT 0.773903 0.810541
32PC0616:32PC0616.MEAS 0.811161 0.884457
32PC0617:32PC0617.MEAS 0.709315 0.786780
32PC0676:32PC0676.OUT 0.753650 0.800376
32FI0648:32FA0648.PNT 0.744748 0.827054
32PC0635:32PC0635.OUT 0.740775 0.812010
32TC0727:32TC0727.MEAS 0.498776 0.628821
PM2SteamKpphTon -0.094431 -0.168900
40MAINS:40_0102.PNT_3 -0.218506 -0.208023
32PC0623:32PB0623.OUT 32PC0623:32PC0623.MEAS ... \
32FC0121:32FC0121.MEAS -0.437463 -0.406653 ...
32FI0607:32FA0607.PNT 0.760814 0.810911 ...
32FI0610:32FA0610.PNT -0.262171 -0.190003 ...
32PC0611:32PC0611.MEAS 0.726499 0.799441 ...
32PC0611:32PC0611.OUT -0.309208 -0.489726 ...
32PC0613:32PC0613.MEAS 0.752780 0.884394 ...
32PC0614:32PC0614.MEAS 0.625777 0.810673 ...
32PC0615:32PC0615.MEAS 0.751950 0.883806 ...
32PC0623:32PB0623.OUT 1.000000 0.887978 ...
32PC0623:32PC0623.MEAS 0.887978 1.000000 ...
32PC0635:32PB0635.OUT 0.858823 0.908982 ...
32PC0635:32PC0635.MEAS 0.890310 0.998072 ...
32PC0665:32PB0665.OUT 0.899641 0.943043 ...
32PC0665:32PC0665.MEAS 0.884707 0.996232 ...
32PC0676:32PB0676.OUT 0.842614 0.916563 ...
32PC0676:32PC0676.MEAS 0.874008 0.989999 ...
32PI0609:32PA0609.PNT -0.323799 -0.203113 ...
32PI0625:32PA0625.PNT -0.294540 -0.207869 ...
40PC0060:40PA0060.PNT -0.174587 -0.043471 ...
40TC0074:40TC0074.MEAS -0.010992 0.045133 ...
32PC0616:32PC0616.OUT -0.727739 -0.841372 ...
32PC0676:32PA0676.PNT 0.874003 0.990002 ...
32FI0637:32FA0637.PNT -0.274542 -0.460291 ...
32FI0650:32FA0650.PNT 0.776339 0.900540 ...
32PC0616:32PC0616.MEAS 0.886767 0.999919 ...
32PC0617:32PC0617.MEAS 0.868069 0.906883 ...
32PC0676:32PC0676.OUT 0.840572 0.913285 ...
32FI0648:32FA0648.PNT 0.853115 0.904482 ...
32PC0635:32PC0635.OUT 0.858443 0.908619 ...
32TC0727:32TC0727.MEAS 0.682141 0.654690 ...
PM2SteamKpphTon -0.276867 -0.211459 ...
40MAINS:40_0102.PNT_3 -0.339863 -0.239874 ...
32FI0637:32FA0637.PNT 32FI0650:32FA0650.PNT \
32FC0121:32FC0121.MEAS 0.083134 -0.371644
32FI0607:32FA0607.PNT -0.087661 0.840685
32FI0610:32FA0610.PNT -0.143325 -0.264727
32PC0611:32PC0611.MEAS -0.048917 0.767864
32PC0611:32PC0611.OUT 0.956610 -0.362271
32PC0613:32PC0613.MEAS -0.274318 0.810740
32PC0614:32PC0614.MEAS -0.277060 0.773903
32PC0615:32PC0615.MEAS -0.273119 0.810541
32PC0623:32PB0623.OUT -0.274542 0.776339
32PC0623:32PC0623.MEAS -0.460291 0.900540
32PC0635:32PB0635.OUT -0.293818 0.853315
32PC0635:32PC0635.MEAS -0.452853 0.904738
32PC0665:32PB0665.OUT -0.410403 0.862510
32PC0665:32PC0665.MEAS -0.445100 0.910737
32PC0676:32PB0676.OUT -0.388799 0.953267
32PC0676:32PC0676.MEAS -0.435466 0.919855
32PI0609:32PA0609.PNT -0.077958 -0.268134
32PI0625:32PA0625.PNT 0.090025 -0.154555
40PC0060:40PA0060.PNT -0.106140 -0.112339
40TC0074:40TC0074.MEAS -0.078211 0.054503
32PC0616:32PC0616.OUT 0.319230 -0.798666
32PC0676:32PA0676.PNT -0.435508 0.919856
32FI0637:32FA0637.PNT 1.000000 -0.344352
32FI0650:32FA0650.PNT -0.344352 1.000000
32PC0616:32PC0616.MEAS -0.462991 0.899308
32PC0617:32PC0617.MEAS -0.453722 0.790501
32PC0676:32PC0676.OUT -0.387393 0.951988
32FI0648:32FA0648.PNT -0.204703 0.867638
32PC0635:32PC0635.OUT -0.293932 0.853207
32TC0727:32TC0727.MEAS -0.290457 0.586394
PM2SteamKpphTon 0.226581 -0.111615
40MAINS:40_0102.PNT_3 0.078087 -0.200262
32PC0616:32PC0616.MEAS 32PC0617:32PC0617.MEAS \
32FC0121:32FC0121.MEAS -0.404226 -0.437322
32FI0607:32FA0607.PNT 0.807063 0.685285
32FI0610:32FA0610.PNT -0.188644 -0.217246
32PC0611:32PC0611.MEAS 0.798990 0.716719
32PC0611:32PC0611.OUT -0.491503 -0.487945
32PC0613:32PC0613.MEAS 0.885037 0.787477
32PC0614:32PC0614.MEAS 0.811161 0.709315
32PC0615:32PC0615.MEAS 0.884457 0.786780
32PC0623:32PB0623.OUT 0.886767 0.868069
32PC0623:32PC0623.MEAS 0.999919 0.906883
32PC0635:32PB0635.OUT 0.908180 0.787626
32PC0635:32PC0635.MEAS 0.997926 0.905246
32PC0665:32PB0665.OUT 0.942247 0.877127
32PC0665:32PC0665.MEAS 0.996087 0.901876
32PC0676:32PB0676.OUT 0.915462 0.823193
32PC0676:32PC0676.MEAS 0.989882 0.898109
32PI0609:32PA0609.PNT -0.202100 -0.184869
32PI0625:32PA0625.PNT -0.208304 -0.243499
40PC0060:40PA0060.PNT -0.043176 -0.043693
40TC0074:40TC0074.MEAS 0.044254 -0.012174
32PC0616:32PC0616.OUT -0.841573 -0.749024
32PC0676:32PA0676.PNT 0.989886 0.898149
32FI0637:32FA0637.PNT -0.462991 -0.453722
32FI0650:32FA0650.PNT 0.899308 0.790501
32PC0616:32PC0616.MEAS 1.000000 0.907022
32PC0617:32PC0617.MEAS 0.907022 1.000000
32PC0676:32PC0676.OUT 0.912166 0.820085
32FI0648:32FA0648.PNT 0.903498 0.788503
32PC0635:32PC0635.OUT 0.907818 0.787203
32TC0727:32TC0727.MEAS 0.657491 0.693768
PM2SteamKpphTon -0.214008 -0.305620
40MAINS:40_0102.PNT_3 -0.240453 -0.287813
32PC0676:32PC0676.OUT 32FI0648:32FA0648.PNT \
32FC0121:32FC0121.MEAS -0.343931 -0.363981
32FI0607:32FA0607.PNT 0.828213 0.847098
32FI0610:32FA0610.PNT -0.216996 -0.287463
32PC0611:32PC0611.MEAS 0.735180 0.792893
32PC0611:32PC0611.OUT -0.378932 -0.232162
32PC0613:32PC0613.MEAS 0.800554 0.827481
32PC0614:32PC0614.MEAS 0.753650 0.744748
32PC0615:32PC0615.MEAS 0.800376 0.827054
32PC0623:32PB0623.OUT 0.840572 0.853115
32PC0623:32PC0623.MEAS 0.913285 0.904482
32PC0635:32PB0635.OUT 0.899521 0.942128
32PC0635:32PC0635.MEAS 0.916859 0.910224
32PC0665:32PB0665.OUT 0.920798 0.860587
32PC0665:32PC0665.MEAS 0.917030 0.912417
32PC0676:32PB0676.OUT 0.999195 0.868249
32PC0676:32PC0676.MEAS 0.917755 0.912587
32PI0609:32PA0609.PNT -0.385812 -0.272266
32PI0625:32PA0625.PNT -0.211117 -0.219040
40PC0060:40PA0060.PNT -0.232939 -0.118875
40TC0074:40TC0074.MEAS 0.022176 -0.054141
32PC0616:32PC0616.OUT -0.755411 -0.807663
32PC0676:32PA0676.PNT 0.917755 0.912571
32FI0637:32FA0637.PNT -0.387393 -0.204703
32FI0650:32FA0650.PNT 0.951988 0.867638
32PC0616:32PC0616.MEAS 0.912166 0.903498
32PC0617:32PC0617.MEAS 0.820085 0.788503
32PC0676:32PC0676.OUT 1.000000 0.866576
32FI0648:32FA0648.PNT 0.866576 1.000000
32PC0635:32PC0635.OUT 0.899354 0.941891
32TC0727:32TC0727.MEAS 0.581667 0.578755
PM2SteamKpphTon -0.109768 -0.143827
40MAINS:40_0102.PNT_3 -0.250766 -0.266355
32PC0635:32PC0635.OUT 32TC0727:32TC0727.MEAS \
32FC0121:32FC0121.MEAS -0.303126 -0.406059
32FI0607:32FA0607.PNT 0.822049 0.359103
32FI0610:32FA0610.PNT -0.180220 -0.341058
32PC0611:32PC0611.MEAS 0.754611 0.620756
32PC0611:32PC0611.OUT -0.288992 -0.312901
32PC0613:32PC0613.MEAS 0.812345 0.629345
32PC0614:32PC0614.MEAS 0.740775 0.498776
32PC0615:32PC0615.MEAS 0.812010 0.628821
32PC0623:32PB0623.OUT 0.858443 0.682141
32PC0623:32PC0623.MEAS 0.908619 0.654690
32PC0635:32PB0635.OUT 0.999811 0.533626
32PC0635:32PC0635.MEAS 0.910409 0.662669
32PC0665:32PB0665.OUT 0.891610 0.641563
32PC0665:32PC0665.MEAS 0.908331 0.668733
32PC0676:32PB0676.OUT 0.900510 0.584132
32PC0676:32PC0676.MEAS 0.900485 0.677082
32PI0609:32PA0609.PNT -0.352557 -0.121152
32PI0625:32PA0625.PNT -0.230722 -0.188258
40PC0060:40PA0060.PNT -0.201331 -0.046134
40TC0074:40TC0074.MEAS -0.017308 -0.012509
32PC0616:32PC0616.OUT -0.761828 -0.596753
32PC0676:32PA0676.PNT 0.900482 0.677052
32FI0637:32FA0637.PNT -0.293932 -0.290457
32FI0650:32FA0650.PNT 0.853207 0.586394
32PC0616:32PC0616.MEAS 0.907818 0.657491
32PC0617:32PC0617.MEAS 0.787203 0.693768
32PC0676:32PC0676.OUT 0.899354 0.581667
32FI0648:32FA0648.PNT 0.941891 0.578755
32PC0635:32PC0635.OUT 1.000000 0.532955
32TC0727:32TC0727.MEAS 0.532955 1.000000
PM2SteamKpphTon -0.098365 -0.548471
40MAINS:40_0102.PNT_3 -0.261130 -0.245659
PM2SteamKpphTon 40MAINS:40_0102.PNT_3
32FC0121:32FC0121.MEAS 0.270986 0.272100
32FI0607:32FA0607.PNT 0.129926 -0.217564
32FI0610:32FA0610.PNT 0.215561 0.293495
32PC0611:32PC0611.MEAS -0.160520 -0.203608
32PC0611:32PC0611.OUT 0.254943 0.095499
32PC0613:32PC0613.MEAS -0.168569 -0.207388
32PC0614:32PC0614.MEAS -0.094431 -0.218506
32PC0615:32PC0615.MEAS -0.168900 -0.208023
32PC0623:32PB0623.OUT -0.276867 -0.339863
32PC0623:32PC0623.MEAS -0.211459 -0.239874
32PC0635:32PB0635.OUT -0.099257 -0.260898
32PC0635:32PC0635.MEAS -0.217318 -0.243270
32PC0665:32PB0665.OUT -0.222089 -0.361180
32PC0665:32PC0665.MEAS -0.218513 -0.241652
32PC0676:32PB0676.OUT -0.115669 -0.252122
32PC0676:32PC0676.MEAS -0.222136 -0.243330
32PI0609:32PA0609.PNT 0.036076 0.384344
32PI0625:32PA0625.PNT 0.169827 0.945702
40PC0060:40PA0060.PNT 0.054013 0.335356
40TC0074:40TC0074.MEAS 0.074987 0.249328
32PC0616:32PC0616.OUT 0.230930 0.211033
32PC0676:32PA0676.PNT -0.222141 -0.243405
32FI0637:32FA0637.PNT 0.226581 0.078087
32FI0650:32FA0650.PNT -0.111615 -0.200262
32PC0616:32PC0616.MEAS -0.214008 -0.240453
32PC0617:32PC0617.MEAS -0.305620 -0.287813
32PC0676:32PC0676.OUT -0.109768 -0.250766
32FI0648:32FA0648.PNT -0.143827 -0.266355
32PC0635:32PC0635.OUT -0.098365 -0.261130
32TC0727:32TC0727.MEAS -0.548471 -0.245659
PM2SteamKpphTon 1.000000 0.218567
40MAINS:40_0102.PNT_3 0.218567 1.000000
[32 rows x 32 columns]
Filled missing values in 32FI0610:32FA0610.PNT using Random Forest.
Filled missing values in 40PC0060:40PA0060.PNT using Random Forest.
Filled missing values in 40TC0074:40TC0074.MEAS using Random Forest.
Filled missing values in 32PC0616:32PC0616.OUT using Random Forest.
Filled missing values in 32FI0650:32FA0650.PNT using Random Forest.
Filled missing values in 32PC0617:32PC0617.MEAS using Random Forest.
Filled missing values in 32FI0648:32FA0648.PNT using Random Forest.
Filled missing values in 32TC0727:32TC0727.MEAS using Random Forest.
Filled missing values in PM2SteamKpphTon using Random Forest.
Filled missing values in 40MAINS:40_0102.PNT_3 using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32FC0121:32FC0121.MEAS', '32FI0607:32FA0607.PNT', '32FI0610:32FA0610.PNT', '32PC0611:32PC0611.MEAS', '32PC0611:32PC0611.OUT', '32PC0613:32PC0613.MEAS', '32PC0614:32PC0614.MEAS', '32PC0615:32PC0615.MEAS', '32PC0623:32PB0623.OUT', '32PC0623:32PC0623.MEAS', '32PC0635:32PB0635.OUT', '32PC0635:32PC0635.MEAS', '32PC0665:32PB0665.OUT', '32PC0665:32PC0665.MEAS', '32PC0676:32PB0676.OUT', '32PC0676:32PC0676.MEAS', '32PI0609:32PA0609.PNT', '32PI0625:32PA0625.PNT', '40PC0060:40PA0060.PNT', '40TC0074:40TC0074.MEAS', '32PC0616:32PC0616.OUT', '32PC0676:32PA0676.PNT', '32FI0637:32FA0637.PNT', '32FI0650:32FA0650.PNT', '32PC0616:32PC0616.MEAS', '32PC0617:32PC0617.MEAS', '32PC0676:32PC0676.OUT', '32FI0648:32FA0648.PNT', '32PC0635:32PC0635.OUT', '32TC0727:32TC0727.MEAS', 'PM2SteamKpphTon', '40MAINS:40_0102.PNT_3']
Filled missing values in 32FI0610:32FA0610.PNT using forward and backward fill.
Filled missing values in 40PC0060:40PA0060.PNT using forward and backward fill.
Filled missing values in 40TC0074:40TC0074.MEAS using forward and backward fill.
Filled missing values in 32PC0616:32PC0616.OUT using forward and backward fill.
Filled missing values in 32FI0650:32FA0650.PNT using forward and backward fill.
Filled missing values in 32PC0617:32PC0617.MEAS using forward and backward fill.
Filled missing values in 32FI0648:32FA0648.PNT using forward and backward fill.
Filled missing values in 32TC0727:32TC0727.MEAS using forward and backward fill.
Filled missing values in PM2SteamKpphTon using forward and backward fill.
Filled missing values in 40MAINS:40_0102.PNT_3 using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Control:
32HC0080:32HF0080.PNT 40PC0060:40PC0052.MEAS \
32HC0080:32HF0080.PNT 1.000000 -0.226162
40PC0060:40PC0052.MEAS -0.226162 1.000000
32LC0682:32LC0682.OUT 0.192133 -0.048556
29HC0094:29HA0094.PNT 0.844182 -0.249967
32PC0164:32PCD164.MEAS -0.386124 0.065404
27NC0021:27NC0021.MEAS 0.159541 -0.084735
27NC0061:27NC0061.MEAS -0.023186 -0.004579
32LC0682:32LC0682.OUT 29HC0094:29HA0094.PNT \
32HC0080:32HF0080.PNT 0.192133 0.844182
40PC0060:40PC0052.MEAS -0.048556 -0.249967
32LC0682:32LC0682.OUT 1.000000 0.207465
29HC0094:29HA0094.PNT 0.207465 1.000000
32PC0164:32PCD164.MEAS -0.106715 -0.370602
27NC0021:27NC0021.MEAS 0.042114 0.149063
27NC0061:27NC0061.MEAS -0.031313 0.014655
32PC0164:32PCD164.MEAS 27NC0021:27NC0021.MEAS \
32HC0080:32HF0080.PNT -0.386124 0.159541
40PC0060:40PC0052.MEAS 0.065404 -0.084735
32LC0682:32LC0682.OUT -0.106715 0.042114
29HC0094:29HA0094.PNT -0.370602 0.149063
32PC0164:32PCD164.MEAS 1.000000 -0.170783
27NC0021:27NC0021.MEAS -0.170783 1.000000
27NC0061:27NC0061.MEAS 0.043600 -0.017598
27NC0061:27NC0061.MEAS
32HC0080:32HF0080.PNT -0.023186
40PC0060:40PC0052.MEAS -0.004579
32LC0682:32LC0682.OUT -0.031313
29HC0094:29HA0094.PNT 0.014655
32PC0164:32PCD164.MEAS 0.043600
27NC0021:27NC0021.MEAS -0.017598
27NC0061:27NC0061.MEAS 1.000000
Filled missing values in 32HC0080:32HF0080.PNT using Random Forest.
Filled missing values in 40PC0060:40PC0052.MEAS using Random Forest.
Filled missing values in 32PC0164:32PCD164.MEAS using Random Forest.
Filled missing values in 27NC0021:27NC0021.MEAS using Random Forest.
Filled missing values in 27NC0061:27NC0061.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32HC0080:32HF0080.PNT', '40PC0060:40PC0052.MEAS', '32LC0682:32LC0682.OUT', '29HC0094:29HA0094.PNT', '32PC0164:32PCD164.MEAS', '27NC0021:27NC0021.MEAS', '27NC0061:27NC0061.MEAS']
Filled missing values in 32HC0080:32HF0080.PNT using forward and backward fill.
Filled missing values in 40PC0060:40PC0052.MEAS using forward and backward fill.
Filled missing values in 32PC0164:32PCD164.MEAS using forward and backward fill.
Filled missing values in 27NC0021:27NC0021.MEAS using forward and backward fill.
Filled missing values in 27NC0061:27NC0061.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Vacuum:
32II0171Z:32IA0171Z.PNT 32II0179Z:32IA0179Z.PNT \
32II0171Z:32IA0171Z.PNT 1.000000 0.386157
32II0179Z:32IA0179Z.PNT 0.386157 1.000000
32II0439A:32IA0439A.PNT -0.133437 0.157455
32PC0146:32PC0146.MEAS -0.077862 0.059270
32PC0163:32PC0163.MEAS -0.130829 -0.027058
32PC0164:32PC0164.MEAS -0.187793 -0.071281
32PC0179:32PC0179.MEAS -0.202422 0.023029
32PC0532:32PC0532.MEAS -0.293100 -0.391496
32PC0532:32PC0532.OUT 0.029967 0.366521
32PC0905:32PC0905.MEAS 0.052173 -0.329996
32PI0035:32PA0035.PNT 0.312416 0.103650
32PC0905:32PC0905.OUT 0.056977 0.161674
32II0439A:32IA0439A.PNT 32PC0146:32PC0146.MEAS \
32II0171Z:32IA0171Z.PNT -0.133437 -0.077862
32II0179Z:32IA0179Z.PNT 0.157455 0.059270
32II0439A:32IA0439A.PNT 1.000000 -0.040749
32PC0146:32PC0146.MEAS -0.040749 1.000000
32PC0163:32PC0163.MEAS -0.000221 0.004937
32PC0164:32PC0164.MEAS 0.027614 0.009320
32PC0179:32PC0179.MEAS 0.052630 -0.098286
32PC0532:32PC0532.MEAS 0.186717 -0.020317
32PC0532:32PC0532.OUT 0.303337 0.044485
32PC0905:32PC0905.MEAS -0.990523 0.051957
32PI0035:32PA0035.PNT -0.071085 -0.056282
32PC0905:32PC0905.OUT 0.467753 -0.014520
32PC0163:32PC0163.MEAS 32PC0164:32PC0164.MEAS \
32II0171Z:32IA0171Z.PNT -0.130829 -0.187793
32II0179Z:32IA0179Z.PNT -0.027058 -0.071281
32II0439A:32IA0439A.PNT -0.000221 0.027614
32PC0146:32PC0146.MEAS 0.004937 0.009320
32PC0163:32PC0163.MEAS 1.000000 0.747933
32PC0164:32PC0164.MEAS 0.747933 1.000000
32PC0179:32PC0179.MEAS -0.017710 0.037260
32PC0532:32PC0532.MEAS 0.064071 0.065979
32PC0532:32PC0532.OUT -0.053572 -0.038451
32PC0905:32PC0905.MEAS 0.005399 -0.040774
32PI0035:32PA0035.PNT -0.038473 -0.011573
32PC0905:32PC0905.OUT -0.025355 -0.044297
32PC0179:32PC0179.MEAS 32PC0532:32PC0532.MEAS \
32II0171Z:32IA0171Z.PNT -0.202422 -0.293100
32II0179Z:32IA0179Z.PNT 0.023029 -0.391496
32II0439A:32IA0439A.PNT 0.052630 0.186717
32PC0146:32PC0146.MEAS -0.098286 -0.020317
32PC0163:32PC0163.MEAS -0.017710 0.064071
32PC0164:32PC0164.MEAS 0.037260 0.065979
32PC0179:32PC0179.MEAS 1.000000 0.077139
32PC0532:32PC0532.MEAS 0.077139 1.000000
32PC0532:32PC0532.OUT 0.230712 -0.342267
32PC0905:32PC0905.MEAS -0.070313 -0.027408
32PI0035:32PA0035.PNT 0.098994 0.024269
32PC0905:32PC0905.OUT -0.133655 0.000176
32PC0532:32PC0532.OUT 32PC0905:32PC0905.MEAS \
32II0171Z:32IA0171Z.PNT 0.029967 0.052173
32II0179Z:32IA0179Z.PNT 0.366521 -0.329996
32II0439A:32IA0439A.PNT 0.303337 -0.990523
32PC0146:32PC0146.MEAS 0.044485 0.051957
32PC0163:32PC0163.MEAS -0.053572 0.005399
32PC0164:32PC0164.MEAS -0.038451 -0.040774
32PC0179:32PC0179.MEAS 0.230712 -0.070313
32PC0532:32PC0532.MEAS -0.342267 -0.027408
32PC0532:32PC0532.OUT 1.000000 -0.337452
32PC0905:32PC0905.MEAS -0.337452 1.000000
32PI0035:32PA0035.PNT -0.417323 -0.009644
32PC0905:32PC0905.OUT 0.006049 -0.567599
32PI0035:32PA0035.PNT 32PC0905:32PC0905.OUT
32II0171Z:32IA0171Z.PNT 0.312416 0.056977
32II0179Z:32IA0179Z.PNT 0.103650 0.161674
32II0439A:32IA0439A.PNT -0.071085 0.467753
32PC0146:32PC0146.MEAS -0.056282 -0.014520
32PC0163:32PC0163.MEAS -0.038473 -0.025355
32PC0164:32PC0164.MEAS -0.011573 -0.044297
32PC0179:32PC0179.MEAS 0.098994 -0.133655
32PC0532:32PC0532.MEAS 0.024269 0.000176
32PC0532:32PC0532.OUT -0.417323 0.006049
32PC0905:32PC0905.MEAS -0.009644 -0.567599
32PI0035:32PA0035.PNT 1.000000 0.114182
32PC0905:32PC0905.OUT 0.114182 1.000000
Filled missing values in 32II0439A:32IA0439A.PNT using Random Forest.
Filled missing values in 32PC0532:32PC0532.MEAS using Random Forest.
Filled missing values in 32PC0905:32PC0905.MEAS using Random Forest.
Filled missing values in 32PC0905:32PC0905.OUT using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32II0171Z:32IA0171Z.PNT', '32II0179Z:32IA0179Z.PNT', '32II0439A:32IA0439A.PNT', '32PC0146:32PC0146.MEAS', '32PC0163:32PC0163.MEAS', '32PC0164:32PC0164.MEAS', '32PC0179:32PC0179.MEAS', '32PC0532:32PC0532.MEAS', '32PC0532:32PC0532.OUT', '32PC0905:32PC0905.MEAS', '32PI0035:32PA0035.PNT', '32PC0905:32PC0905.OUT']
Filled missing values in 32II0439A:32IA0439A.PNT using forward and backward fill.
Filled missing values in 32PC0532:32PC0532.MEAS using forward and backward fill.
Filled missing values in 32PC0905:32PC0905.MEAS using forward and backward fill.
Filled missing values in 32PC0905:32PC0905.OUT using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Agitator & Pulper:
28IT0013:28IT164A.PNT 29PC0147:29PC0147.MEAS \
28IT0013:28IT164A.PNT 1.000000 -0.064596
29PC0147:29PC0147.MEAS -0.064596 1.000000
32LC0274:32LC0274.MEAS.12hrmin 0.048493 0.031552
32NC0268:32NC0268.MEAS 0.031694 -0.148838
32NC0273:32NA0273.PNT 0.011468 -0.016186
32NC0273:32NC0273.MEAS 0.011242 -0.015309
32PC0266:32PC0266.MEAS -0.045398 -0.001831
CONT_PULP:2_TN_HOUR.RO01 0.096695 0.057020
CONT_PULP:2_TN_HOUR.RO02 0.027986 -0.033614
CONT_PULP:2_TN_HOUR.RO03 0.069368 -0.014182
27FI0023:27FI0023.PNT 0.068927 -0.041364
27FI0043:27FI0043.PNT 0.061275 -0.041407
27FI0063:27FI0063.PNT 0.071607 0.016531
32LC0274:32LC0274.MEAS.12hrmin \
28IT0013:28IT164A.PNT 0.048493
29PC0147:29PC0147.MEAS 0.031552
32LC0274:32LC0274.MEAS.12hrmin 1.000000
32NC0268:32NC0268.MEAS -0.049294
32NC0273:32NA0273.PNT 0.185809
32NC0273:32NC0273.MEAS 0.186424
32PC0266:32PC0266.MEAS -0.040620
CONT_PULP:2_TN_HOUR.RO01 -0.155170
CONT_PULP:2_TN_HOUR.RO02 0.016071
CONT_PULP:2_TN_HOUR.RO03 -0.076995
27FI0023:27FI0023.PNT -0.117675
27FI0043:27FI0043.PNT 0.038698
27FI0063:27FI0063.PNT -0.090815
32NC0268:32NC0268.MEAS 32NC0273:32NA0273.PNT \
28IT0013:28IT164A.PNT 0.031694 0.011468
29PC0147:29PC0147.MEAS -0.148838 -0.016186
32LC0274:32LC0274.MEAS.12hrmin -0.049294 0.185809
32NC0268:32NC0268.MEAS 1.000000 0.146748
32NC0273:32NA0273.PNT 0.146748 1.000000
32NC0273:32NC0273.MEAS 0.147326 0.998963
32PC0266:32PC0266.MEAS 0.063408 -0.090680
CONT_PULP:2_TN_HOUR.RO01 0.091508 -0.064182
CONT_PULP:2_TN_HOUR.RO02 0.008627 0.018108
CONT_PULP:2_TN_HOUR.RO03 0.024006 -0.046556
27FI0023:27FI0023.PNT 0.056510 -0.056845
27FI0043:27FI0043.PNT 0.024743 -0.003238
27FI0063:27FI0063.PNT -0.002496 -0.082412
32NC0273:32NC0273.MEAS \
28IT0013:28IT164A.PNT 0.011242
29PC0147:29PC0147.MEAS -0.015309
32LC0274:32LC0274.MEAS.12hrmin 0.186424
32NC0268:32NC0268.MEAS 0.147326
32NC0273:32NA0273.PNT 0.998963
32NC0273:32NC0273.MEAS 1.000000
32PC0266:32PC0266.MEAS -0.090796
CONT_PULP:2_TN_HOUR.RO01 -0.064680
CONT_PULP:2_TN_HOUR.RO02 0.018497
CONT_PULP:2_TN_HOUR.RO03 -0.046402
27FI0023:27FI0023.PNT -0.057147
27FI0043:27FI0043.PNT -0.002904
27FI0063:27FI0063.PNT -0.082269
32PC0266:32PC0266.MEAS \
28IT0013:28IT164A.PNT -0.045398
29PC0147:29PC0147.MEAS -0.001831
32LC0274:32LC0274.MEAS.12hrmin -0.040620
32NC0268:32NC0268.MEAS 0.063408
32NC0273:32NA0273.PNT -0.090680
32NC0273:32NC0273.MEAS -0.090796
32PC0266:32PC0266.MEAS 1.000000
CONT_PULP:2_TN_HOUR.RO01 0.034356
CONT_PULP:2_TN_HOUR.RO02 -0.055563
CONT_PULP:2_TN_HOUR.RO03 0.034082
27FI0023:27FI0023.PNT 0.018446
27FI0043:27FI0043.PNT -0.056231
27FI0063:27FI0063.PNT 0.027424
CONT_PULP:2_TN_HOUR.RO01 \
28IT0013:28IT164A.PNT 0.096695
29PC0147:29PC0147.MEAS 0.057020
32LC0274:32LC0274.MEAS.12hrmin -0.155170
32NC0268:32NC0268.MEAS 0.091508
32NC0273:32NA0273.PNT -0.064182
32NC0273:32NC0273.MEAS -0.064680
32PC0266:32PC0266.MEAS 0.034356
CONT_PULP:2_TN_HOUR.RO01 1.000000
CONT_PULP:2_TN_HOUR.RO02 -0.037122
CONT_PULP:2_TN_HOUR.RO03 0.001328
27FI0023:27FI0023.PNT 0.685126
27FI0043:27FI0043.PNT 0.008034
27FI0063:27FI0063.PNT 0.036636
CONT_PULP:2_TN_HOUR.RO02 \
28IT0013:28IT164A.PNT 0.027986
29PC0147:29PC0147.MEAS -0.033614
32LC0274:32LC0274.MEAS.12hrmin 0.016071
32NC0268:32NC0268.MEAS 0.008627
32NC0273:32NA0273.PNT 0.018108
32NC0273:32NC0273.MEAS 0.018497
32PC0266:32PC0266.MEAS -0.055563
CONT_PULP:2_TN_HOUR.RO01 -0.037122
CONT_PULP:2_TN_HOUR.RO02 1.000000
CONT_PULP:2_TN_HOUR.RO03 0.124785
27FI0023:27FI0023.PNT 0.233228
27FI0043:27FI0043.PNT 0.969824
27FI0063:27FI0063.PNT 0.161303
CONT_PULP:2_TN_HOUR.RO03 \
28IT0013:28IT164A.PNT 0.069368
29PC0147:29PC0147.MEAS -0.014182
32LC0274:32LC0274.MEAS.12hrmin -0.076995
32NC0268:32NC0268.MEAS 0.024006
32NC0273:32NA0273.PNT -0.046556
32NC0273:32NC0273.MEAS -0.046402
32PC0266:32PC0266.MEAS 0.034082
CONT_PULP:2_TN_HOUR.RO01 0.001328
CONT_PULP:2_TN_HOUR.RO02 0.124785
CONT_PULP:2_TN_HOUR.RO03 1.000000
27FI0023:27FI0023.PNT 0.280549
27FI0043:27FI0043.PNT 0.298721
27FI0063:27FI0063.PNT 0.952147
27FI0023:27FI0023.PNT 27FI0043:27FI0043.PNT \
28IT0013:28IT164A.PNT 0.068927 0.061275
29PC0147:29PC0147.MEAS -0.041364 -0.041407
32LC0274:32LC0274.MEAS.12hrmin -0.117675 0.038698
32NC0268:32NC0268.MEAS 0.056510 0.024743
32NC0273:32NA0273.PNT -0.056845 -0.003238
32NC0273:32NC0273.MEAS -0.057147 -0.002904
32PC0266:32PC0266.MEAS 0.018446 -0.056231
CONT_PULP:2_TN_HOUR.RO01 0.685126 0.008034
CONT_PULP:2_TN_HOUR.RO02 0.233228 0.969824
CONT_PULP:2_TN_HOUR.RO03 0.280549 0.298721
27FI0023:27FI0023.PNT 1.000000 0.601791
27FI0043:27FI0043.PNT 0.601791 1.000000
27FI0063:27FI0063.PNT 0.358798 0.371571
27FI0063:27FI0063.PNT
28IT0013:28IT164A.PNT 0.071607
29PC0147:29PC0147.MEAS 0.016531
32LC0274:32LC0274.MEAS.12hrmin -0.090815
32NC0268:32NC0268.MEAS -0.002496
32NC0273:32NA0273.PNT -0.082412
32NC0273:32NC0273.MEAS -0.082269
32PC0266:32PC0266.MEAS 0.027424
CONT_PULP:2_TN_HOUR.RO01 0.036636
CONT_PULP:2_TN_HOUR.RO02 0.161303
CONT_PULP:2_TN_HOUR.RO03 0.952147
27FI0023:27FI0023.PNT 0.358798
27FI0043:27FI0043.PNT 0.371571
27FI0063:27FI0063.PNT 1.000000
Filled missing values in 28IT0013:28IT164A.PNT using Random Forest.
Filled missing values in 32LC0274:32LC0274.MEAS.12hrmin using Random Forest.
Filled missing values in 32NC0273:32NA0273.PNT using Random Forest.
Filled missing values in 32NC0273:32NC0273.MEAS using Random Forest.
Filled missing values in 32PC0266:32PC0266.MEAS using Random Forest.
Filled missing values in CONT_PULP:2_TN_HOUR.RO01 using Random Forest.
Filled missing values in CONT_PULP:2_TN_HOUR.RO02 using Random Forest.
Filled missing values in CONT_PULP:2_TN_HOUR.RO03 using Random Forest.
Filled missing values in 27FI0023:27FI0023.PNT using Random Forest.
Filled missing values in 27FI0043:27FI0043.PNT using Random Forest.
Filled missing values in 27FI0063:27FI0063.PNT using Random Forest.
Filled missing values in medium correlation columns using MICE: ['28IT0013:28IT164A.PNT', '29PC0147:29PC0147.MEAS', '32LC0274:32LC0274.MEAS.12hrmin', '32NC0268:32NC0268.MEAS', '32NC0273:32NA0273.PNT', '32NC0273:32NC0273.MEAS', '32PC0266:32PC0266.MEAS', 'CONT_PULP:2_TN_HOUR.RO01', 'CONT_PULP:2_TN_HOUR.RO02', 'CONT_PULP:2_TN_HOUR.RO03', '27FI0023:27FI0023.PNT', '27FI0043:27FI0043.PNT', '27FI0063:27FI0063.PNT']
Filled missing values in 28IT0013:28IT164A.PNT using forward and backward fill.
Filled missing values in 32LC0274:32LC0274.MEAS.12hrmin using forward and backward fill.
Filled missing values in 32NC0273:32NA0273.PNT using forward and backward fill.
Filled missing values in 32NC0273:32NC0273.MEAS using forward and backward fill.
Filled missing values in 32PC0266:32PC0266.MEAS using forward and backward fill.
Filled missing values in CONT_PULP:2_TN_HOUR.RO01 using forward and backward fill.
Filled missing values in CONT_PULP:2_TN_HOUR.RO02 using forward and backward fill.
Filled missing values in CONT_PULP:2_TN_HOUR.RO03 using forward and backward fill.
Filled missing values in 27FI0023:27FI0023.PNT using forward and backward fill.
Filled missing values in 27FI0043:27FI0043.PNT using forward and backward fill.
Filled missing values in 27FI0063:27FI0063.PNT using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Blend:
19FC0002:19FC0002.MEAS 29NC0107:29NCA107.MEAS \
19FC0002:19FC0002.MEAS 1.000000 -0.093191
29NC0107:29NCA107.MEAS -0.093191 1.000000
29PC0006:29PC0006.MEAS -0.112822 0.260602
32FC1014:32FD1014.RO01 0.016475 -0.161131
32LC0056:32LC0056.MEAS -0.002888 -0.047087
29HC0092:29HA0092.OUT -0.073786 -0.098143
29MCCTEMP:29MCCTEMP.PNT -0.089026 0.167716
29PC0006:29PC0006.OUT -0.033437 0.049266
29PC0077:29PD0077.OUT 0.083554 -0.006913
29FC0073:29FC0073.OUT 0.040736 -0.031085
19NC0001:19NC0001.MEAS -0.007001 0.087185
29PC0006:29PC0006.MEAS 32FC1014:32FD1014.RO01 \
19FC0002:19FC0002.MEAS -0.112822 0.016475
29NC0107:29NCA107.MEAS 0.260602 -0.161131
29PC0006:29PC0006.MEAS 1.000000 -0.178754
32FC1014:32FD1014.RO01 -0.178754 1.000000
32LC0056:32LC0056.MEAS -0.030736 0.008843
29HC0092:29HA0092.OUT -0.415685 -0.031375
29MCCTEMP:29MCCTEMP.PNT -0.198703 0.380509
29PC0006:29PC0006.OUT 0.260224 -0.018304
29PC0077:29PD0077.OUT 0.319350 -0.128496
29FC0073:29FC0073.OUT 0.052886 -0.113462
19NC0001:19NC0001.MEAS 0.211613 -0.173643
32LC0056:32LC0056.MEAS 29HC0092:29HA0092.OUT \
19FC0002:19FC0002.MEAS -0.002888 -0.073786
29NC0107:29NCA107.MEAS -0.047087 -0.098143
29PC0006:29PC0006.MEAS -0.030736 -0.415685
32FC1014:32FD1014.RO01 0.008843 -0.031375
32LC0056:32LC0056.MEAS 1.000000 0.071320
29HC0092:29HA0092.OUT 0.071320 1.000000
29MCCTEMP:29MCCTEMP.PNT -0.005598 -0.037270
29PC0006:29PC0006.OUT -0.016889 -0.149260
29PC0077:29PD0077.OUT -0.039308 -0.536405
29FC0073:29FC0073.OUT 0.033663 -0.102683
19NC0001:19NC0001.MEAS -0.025590 -0.066932
29MCCTEMP:29MCCTEMP.PNT 29PC0006:29PC0006.OUT \
19FC0002:19FC0002.MEAS -0.089026 -0.033437
29NC0107:29NCA107.MEAS 0.167716 0.049266
29PC0006:29PC0006.MEAS -0.198703 0.260224
32FC1014:32FD1014.RO01 0.380509 -0.018304
32LC0056:32LC0056.MEAS -0.005598 -0.016889
29HC0092:29HA0092.OUT -0.037270 -0.149260
29MCCTEMP:29MCCTEMP.PNT 1.000000 -0.072078
29PC0006:29PC0006.OUT -0.072078 1.000000
29PC0077:29PD0077.OUT -0.128341 0.143732
29FC0073:29FC0073.OUT -0.298802 -0.051627
19NC0001:19NC0001.MEAS -0.127824 0.048211
29PC0077:29PD0077.OUT 29FC0073:29FC0073.OUT \
19FC0002:19FC0002.MEAS 0.083554 0.040736
29NC0107:29NCA107.MEAS -0.006913 -0.031085
29PC0006:29PC0006.MEAS 0.319350 0.052886
32FC1014:32FD1014.RO01 -0.128496 -0.113462
32LC0056:32LC0056.MEAS -0.039308 0.033663
29HC0092:29HA0092.OUT -0.536405 -0.102683
29MCCTEMP:29MCCTEMP.PNT -0.128341 -0.298802
29PC0006:29PC0006.OUT 0.143732 -0.051627
29PC0077:29PD0077.OUT 1.000000 -0.105488
29FC0073:29FC0073.OUT -0.105488 1.000000
19NC0001:19NC0001.MEAS 0.119162 -0.085688
19NC0001:19NC0001.MEAS
19FC0002:19FC0002.MEAS -0.007001
29NC0107:29NCA107.MEAS 0.087185
29PC0006:29PC0006.MEAS 0.211613
32FC1014:32FD1014.RO01 -0.173643
32LC0056:32LC0056.MEAS -0.025590
29HC0092:29HA0092.OUT -0.066932
29MCCTEMP:29MCCTEMP.PNT -0.127824
29PC0006:29PC0006.OUT 0.048211
29PC0077:29PD0077.OUT 0.119162
29FC0073:29FC0073.OUT -0.085688
19NC0001:19NC0001.MEAS 1.000000
Filled missing values in 19FC0002:19FC0002.MEAS using Random Forest.
Filled missing values in 32FC1014:32FD1014.RO01 using Random Forest.
Filled missing values in 32LC0056:32LC0056.MEAS using Random Forest.
Filled missing values in 29MCCTEMP:29MCCTEMP.PNT using Random Forest.
Filled missing values in 29PC0077:29PD0077.OUT using Random Forest.
Filled missing values in 29FC0073:29FC0073.OUT using Random Forest.
Filled missing values in 19NC0001:19NC0001.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['19FC0002:19FC0002.MEAS', '29NC0107:29NCA107.MEAS', '29PC0006:29PC0006.MEAS', '32FC1014:32FD1014.RO01', '32LC0056:32LC0056.MEAS', '29HC0092:29HA0092.OUT', '29MCCTEMP:29MCCTEMP.PNT', '29PC0006:29PC0006.OUT', '29PC0077:29PD0077.OUT', '29FC0073:29FC0073.OUT', '19NC0001:19NC0001.MEAS']
Filled missing values in 19FC0002:19FC0002.MEAS using forward and backward fill.
Filled missing values in 32FC1014:32FD1014.RO01 using forward and backward fill.
Filled missing values in 32LC0056:32LC0056.MEAS using forward and backward fill.
Filled missing values in 29MCCTEMP:29MCCTEMP.PNT using forward and backward fill.
Filled missing values in 29PC0077:29PD0077.OUT using forward and backward fill.
Filled missing values in 29FC0073:29FC0073.OUT using forward and backward fill.
Filled missing values in 19NC0001:19NC0001.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Output:
32PICR03:32PACR03.PNT 32LC0119:32LC0119.OUTAVG \
32PICR03:32PACR03.PNT 1.000000 0.052729
32LC0119:32LC0119.OUTAVG 0.052729 1.000000
32LC0274:32LC0274.OUTAVG 0.119582 0.489599
32NC0273:32NE0273.OUT -0.026230 -0.147091
32PC0272:32PC0272.OUTAVG -0.255568 -0.683622
32DC0622:32DC0622.OUTAVG -0.056537 -0.602839
32LC0624:32LC0624.OUTAVG -0.104042 -0.447167
32LC0636:32LC0636.OUTAVG 0.013686 0.189168
32LC0682:32LC0682.OUTAVG -0.305848 -0.124204
32PC0611:32PC0611.OUTAVG -0.081009 -0.403768
32PC0623:32PC0623.OUTAVG -0.087952 -0.007561
32PC0635:32PC0635.OUTAVG -0.042087 0.327317
32PC0676:32PC0676.OUTAVG -0.118750 0.272637
32PC0681:32PC0681.OUTAVG 0.023105 0.122442
32PC0905:32PC0905.OUTAVG -0.412159 -0.592572
32PI13AG:32PA13AG.PNT 0.068025 0.114306
32PC0153:32PC0153.OUTAVG 0.290239 0.315324
32TC15AR:32TC15AR.OUTAVG 0.041977 0.077992
32TC16AR:32TC16AR.OUTAVG 0.255316 -0.288850
32LC0619:32LC0619.OUTAVG -0.017454 0.345847
29FC0074:29FC0074.OUTAVG 0.321276 0.617460
29FC0334:29FC0334.OUTAVG -0.186094 -0.507179
29FCA093:29FC0093.OUTAVG 0.147349 0.562169
29HC0092:29HC0092.OUTAVG -0.072548 0.231274
29LC0071:29LC0071.OUTAVG 0.020788 0.159382
29LC0088:29LC0088.OUTAVG -0.064502 0.229130
29PC0006:29PC0006.OUTAVG -0.112552 -0.244665
29PC0077:29PC0077.OUTAVG -0.146088 0.465984
32LC0015:32LC0015.OUTAVG -0.019831 -0.254144
32PC1012:32PC1012.OUTAVG -0.017595 -0.352746
29PC0147:29PC0147.OUTAVG -0.221074 0.553731
29PC0150:29PC0150.OUTAVG -0.087136 0.454278
32LC0673:32LC0673.OUTAVG 0.311815 0.349349
29FC0073:29FC0073.OUTAVG -0.211184 -0.580052
32FC0677:32FC0677.OUTAVG -0.133456 -0.214528
32TC0803:32TI0803.OUTAVG 0.158009 0.137659
32LC0628:32LC0628.OUTAVG 0.254340 0.660501
32LC0642:32LC0642.OUTAVG -0.003988 0.051220
32DC0679:32DC0679.OUTAVG -0.034241 0.005043
32DC0670:32DC0670.OUTAVG 0.219192 0.337535
32FC0669:32FC0669.OUTAVG 0.047379 -0.046931
32PC0093:32PC0093.OUTAVG 0.054669 0.289715
32LC0100:32LC0100.OUTAVG -0.216792 -0.573117
32TC0081:32TC0081.OUTAVG -0.172643 0.554931
32LC0274:32LC0274.OUTAVG 32NC0273:32NE0273.OUT \
32PICR03:32PACR03.PNT 0.119582 -0.026230
32LC0119:32LC0119.OUTAVG 0.489599 -0.147091
32LC0274:32LC0274.OUTAVG 1.000000 -0.148622
32NC0273:32NE0273.OUT -0.148622 1.000000
32PC0272:32PC0272.OUTAVG -0.748486 0.183386
32DC0622:32DC0622.OUTAVG -0.562751 0.075099
32LC0624:32LC0624.OUTAVG -0.240928 0.237768
32LC0636:32LC0636.OUTAVG 0.316197 -0.050073
32LC0682:32LC0682.OUTAVG -0.070260 0.126693
32PC0611:32PC0611.OUTAVG -0.406385 0.156516
32PC0623:32PC0623.OUTAVG 0.045297 0.118694
32PC0635:32PC0635.OUTAVG 0.340488 0.018290
32PC0676:32PC0676.OUTAVG 0.271097 0.056458
32PC0681:32PC0681.OUTAVG 0.104312 0.072907
32PC0905:32PC0905.OUTAVG -0.599004 0.139931
32PI13AG:32PA13AG.PNT 0.135997 0.247734
32PC0153:32PC0153.OUTAVG 0.512902 -0.140533
32TC15AR:32TC15AR.OUTAVG 0.148305 0.122513
32TC16AR:32TC16AR.OUTAVG -0.076827 0.145478
32LC0619:32LC0619.OUTAVG 0.382934 0.060240
29FC0074:29FC0074.OUTAVG 0.600437 -0.110379
29FC0334:29FC0334.OUTAVG -0.481408 0.081601
29FCA093:29FC0093.OUTAVG 0.587861 -0.051377
29HC0092:29HC0092.OUTAVG 0.331623 0.058885
29LC0071:29LC0071.OUTAVG 0.496129 -0.008226
29LC0088:29LC0088.OUTAVG 0.307958 0.070951
29PC0006:29PC0006.OUTAVG -0.320207 0.024765
29PC0077:29PC0077.OUTAVG 0.231953 -0.041663
32LC0015:32LC0015.OUTAVG -0.069778 -0.012831
32PC1012:32PC1012.OUTAVG -0.340166 0.091396
29PC0147:29PC0147.OUTAVG 0.331129 -0.035964
29PC0150:29PC0150.OUTAVG 0.117448 -0.110277
32LC0673:32LC0673.OUTAVG 0.264088 -0.008602
29FC0073:29FC0073.OUTAVG -0.531466 0.084459
32FC0677:32FC0677.OUTAVG -0.114483 -0.017217
32TC0803:32TI0803.OUTAVG 0.269376 0.006274
32LC0628:32LC0628.OUTAVG 0.618263 -0.110114
32LC0642:32LC0642.OUTAVG 0.093255 0.048937
32DC0679:32DC0679.OUTAVG 0.105932 -0.065931
32DC0670:32DC0670.OUTAVG 0.369496 -0.092657
32FC0669:32FC0669.OUTAVG -0.000065 -0.017864
32PC0093:32PC0093.OUTAVG 0.438569 -0.102805
32LC0100:32LC0100.OUTAVG -0.709061 0.124166
32TC0081:32TC0081.OUTAVG 0.451645 -0.090337
32PC0272:32PC0272.OUTAVG 32DC0622:32DC0622.OUTAVG \
32PICR03:32PACR03.PNT -0.255568 -0.056537
32LC0119:32LC0119.OUTAVG -0.683622 -0.602839
32LC0274:32LC0274.OUTAVG -0.748486 -0.562751
32NC0273:32NE0273.OUT 0.183386 0.075099
32PC0272:32PC0272.OUTAVG 1.000000 0.682452
32DC0622:32DC0622.OUTAVG 0.682452 1.000000
32LC0624:32LC0624.OUTAVG 0.584586 0.136020
32LC0636:32LC0636.OUTAVG -0.243666 -0.446032
32LC0682:32LC0682.OUTAVG 0.342526 -0.198212
32PC0611:32PC0611.OUTAVG 0.510248 0.185799
32PC0623:32PC0623.OUTAVG 0.158176 -0.403446
32PC0635:32PC0635.OUTAVG -0.277394 -0.773705
32PC0676:32PC0676.OUTAVG -0.164804 -0.714035
32PC0681:32PC0681.OUTAVG -0.091494 -0.497645
32PC0905:32PC0905.OUTAVG 0.849345 0.509042
32PI13AG:32PA13AG.PNT -0.215254 -0.088157
32PC0153:32PC0153.OUTAVG -0.543158 -0.284447
32TC15AR:32TC15AR.OUTAVG 0.009693 -0.365771
32TC16AR:32TC16AR.OUTAVG 0.269435 0.167242
32LC0619:32LC0619.OUTAVG -0.305890 -0.785664
29FC0074:29FC0074.OUTAVG -0.848709 -0.652016
29FC0334:29FC0334.OUTAVG 0.682287 0.672446
29FCA093:29FC0093.OUTAVG -0.674431 -0.889807
29HC0092:29HC0092.OUTAVG -0.175006 -0.646087
29LC0071:29LC0071.OUTAVG -0.267221 -0.521704
29LC0088:29LC0088.OUTAVG -0.162171 -0.630328
29PC0006:29PC0006.OUTAVG 0.229291 0.286157
29PC0077:29PC0077.OUTAVG -0.420653 -0.395051
32LC0015:32LC0015.OUTAVG 0.239123 0.367057
32PC1012:32PC1012.OUTAVG 0.337427 0.396261
29PC0147:29PC0147.OUTAVG -0.467214 -0.594786
29PC0150:29PC0150.OUTAVG -0.378239 -0.225820
32LC0673:32LC0673.OUTAVG -0.363343 -0.232810
29FC0073:29FC0073.OUTAVG 0.706181 0.635517
32FC0677:32FC0677.OUTAVG 0.220604 0.299863
32TC0803:32TI0803.OUTAVG -0.291353 -0.286781
32LC0628:32LC0628.OUTAVG -0.845388 -0.822914
32LC0642:32LC0642.OUTAVG -0.083607 -0.350335
32DC0679:32DC0679.OUTAVG -0.094842 0.086085
32DC0670:32DC0670.OUTAVG -0.535680 -0.240620
32FC0669:32FC0669.OUTAVG 0.000083 0.154438
32PC0093:32PC0093.OUTAVG -0.447188 -0.261902
32LC0100:32LC0100.OUTAVG 0.789489 0.773437
32TC0081:32TC0081.OUTAVG -0.600262 -0.647195
32LC0624:32LC0624.OUTAVG 32LC0636:32LC0636.OUTAVG \
32PICR03:32PACR03.PNT -0.104042 0.013686
32LC0119:32LC0119.OUTAVG -0.447167 0.189168
32LC0274:32LC0274.OUTAVG -0.240928 0.316197
32NC0273:32NE0273.OUT 0.237768 -0.050073
32PC0272:32PC0272.OUTAVG 0.584586 -0.243666
32DC0622:32DC0622.OUTAVG 0.136020 -0.446032
32LC0624:32LC0624.OUTAVG 1.000000 -0.000548
32LC0636:32LC0636.OUTAVG -0.000548 1.000000
32LC0682:32LC0682.OUTAVG 0.574103 0.115932
32PC0611:32PC0611.OUTAVG 0.389464 -0.085410
32PC0623:32PC0623.OUTAVG 0.704171 0.124988
32PC0635:32PC0635.OUTAVG 0.307969 0.292529
32PC0676:32PC0676.OUTAVG 0.340887 0.234542
32PC0681:32PC0681.OUTAVG 0.306469 0.005649
32PC0905:32PC0905.OUTAVG 0.504189 -0.170546
32PI13AG:32PA13AG.PNT -0.120137 -0.000192
32PC0153:32PC0153.OUTAVG -0.169568 0.042727
32TC15AR:32TC15AR.OUTAVG 0.502296 0.188034
32TC16AR:32TC16AR.OUTAVG 0.585255 -0.118155
32LC0619:32LC0619.OUTAVG 0.334786 0.416320
29FC0074:29FC0074.OUTAVG -0.430774 0.210929
29FC0334:29FC0334.OUTAVG 0.480481 -0.409915
29FCA093:29FC0093.OUTAVG -0.066118 0.353671
29HC0092:29HC0092.OUTAVG 0.444760 0.387062
29LC0071:29LC0071.OUTAVG 0.281233 0.522163
29LC0088:29LC0088.OUTAVG 0.493129 0.185211
29PC0006:29PC0006.OUTAVG -0.007500 -0.098400
29PC0077:29PC0077.OUTAVG -0.497818 0.110692
32LC0015:32LC0015.OUTAVG 0.161016 -0.129772
32PC1012:32PC1012.OUTAVG 0.058107 -0.209132
29PC0147:29PC0147.OUTAVG -0.403772 0.289637
29PC0150:29PC0150.OUTAVG -0.536968 0.028988
32LC0673:32LC0673.OUTAVG -0.227961 0.058280
29FC0073:29FC0073.OUTAVG 0.413088 -0.343168
32FC0677:32FC0677.OUTAVG 0.162752 0.107058
32TC0803:32TI0803.OUTAVG -0.017792 0.049717
32LC0628:32LC0628.OUTAVG -0.402491 0.270156
32LC0642:32LC0642.OUTAVG 0.370945 0.162552
32DC0679:32DC0679.OUTAVG -0.036391 0.191690
32DC0670:32DC0670.OUTAVG -0.314549 0.312175
32FC0669:32FC0669.OUTAVG 0.029008 0.207125
32PC0093:32PC0093.OUTAVG -0.208077 -0.024312
32LC0100:32LC0100.OUTAVG 0.263969 -0.349889
32TC0081:32TC0081.OUTAVG -0.315244 0.358427
32LC0682:32LC0682.OUTAVG 32PC0611:32PC0611.OUTAVG \
32PICR03:32PACR03.PNT -0.305848 -0.081009
32LC0119:32LC0119.OUTAVG -0.124204 -0.403768
32LC0274:32LC0274.OUTAVG -0.070260 -0.406385
32NC0273:32NE0273.OUT 0.126693 0.156516
32PC0272:32PC0272.OUTAVG 0.342526 0.510248
32DC0622:32DC0622.OUTAVG -0.198212 0.185799
32LC0624:32LC0624.OUTAVG 0.574103 0.389464
32LC0636:32LC0636.OUTAVG 0.115932 -0.085410
32LC0682:32LC0682.OUTAVG 1.000000 0.370427
32PC0611:32PC0611.OUTAVG 0.370427 1.000000
32PC0623:32PC0623.OUTAVG 0.510084 0.164182
32PC0635:32PC0635.OUTAVG 0.445802 -0.078488
32PC0676:32PC0676.OUTAVG 0.476240 -0.047742
32PC0681:32PC0681.OUTAVG 0.360953 0.047947
32PC0905:32PC0905.OUTAVG 0.479237 0.533678
32PI13AG:32PA13AG.PNT -0.165232 -0.125636
32PC0153:32PC0153.OUTAVG -0.183755 -0.468125
32TC15AR:32TC15AR.OUTAVG 0.313308 -0.090708
32TC16AR:32TC16AR.OUTAVG 0.116317 0.000733
32LC0619:32LC0619.OUTAVG 0.523268 0.053291
29FC0074:29FC0074.OUTAVG -0.323386 -0.436760
29FC0334:29FC0334.OUTAVG 0.186617 0.269291
29FCA093:29FC0093.OUTAVG 0.091892 -0.251615
29HC0092:29HC0092.OUTAVG 0.393652 -0.170636
29LC0071:29LC0071.OUTAVG 0.220270 -0.131726
29LC0088:29LC0088.OUTAVG 0.481184 -0.075909
29PC0006:29PC0006.OUTAVG 0.060279 0.014181
29PC0077:29PC0077.OUTAVG -0.101022 0.020009
32LC0015:32LC0015.OUTAVG 0.017014 -0.024083
32PC1012:32PC1012.OUTAVG -0.195054 0.223566
29PC0147:29PC0147.OUTAVG 0.037009 -0.065175
29PC0150:29PC0150.OUTAVG -0.191023 -0.178928
32LC0673:32LC0673.OUTAVG 0.004783 0.022347
29FC0073:29FC0073.OUTAVG 0.209392 0.298129
32FC0677:32FC0677.OUTAVG 0.142755 -0.089302
32TC0803:32TI0803.OUTAVG 0.043038 -0.164714
32LC0628:32LC0628.OUTAVG -0.146512 -0.279592
32LC0642:32LC0642.OUTAVG 0.231706 -0.229093
32DC0679:32DC0679.OUTAVG 0.022625 -0.243294
32DC0670:32DC0670.OUTAVG -0.213730 -0.432285
32FC0669:32FC0669.OUTAVG -0.002653 -0.213937
32PC0093:32PC0093.OUTAVG -0.158080 -0.339803
32LC0100:32LC0100.OUTAVG 0.049758 0.407172
32TC0081:32TC0081.OUTAVG 0.054969 -0.220934
... 32FC0677:32FC0677.OUTAVG \
32PICR03:32PACR03.PNT ... -0.133456
32LC0119:32LC0119.OUTAVG ... -0.214528
32LC0274:32LC0274.OUTAVG ... -0.114483
32NC0273:32NE0273.OUT ... -0.017217
32PC0272:32PC0272.OUTAVG ... 0.220604
32DC0622:32DC0622.OUTAVG ... 0.299863
32LC0624:32LC0624.OUTAVG ... 0.162752
32LC0636:32LC0636.OUTAVG ... 0.107058
32LC0682:32LC0682.OUTAVG ... 0.142755
32PC0611:32PC0611.OUTAVG ... -0.089302
32PC0623:32PC0623.OUTAVG ... -0.253526
32PC0635:32PC0635.OUTAVG ... -0.328284
32PC0676:32PC0676.OUTAVG ... -0.378109
32PC0681:32PC0681.OUTAVG ... -0.540129
32PC0905:32PC0905.OUTAVG ... 0.259014
32PI13AG:32PA13AG.PNT ... -0.145179
32PC0153:32PC0153.OUTAVG ... -0.163525
32TC15AR:32TC15AR.OUTAVG ... -0.020365
32TC16AR:32TC16AR.OUTAVG ... -0.040663
32LC0619:32LC0619.OUTAVG ... -0.301032
29FC0074:29FC0074.OUTAVG ... -0.340177
29FC0334:29FC0334.OUTAVG ... 0.134834
29FCA093:29FC0093.OUTAVG ... -0.397173
29HC0092:29HC0092.OUTAVG ... -0.049367
29LC0071:29LC0071.OUTAVG ... -0.014972
29LC0088:29LC0088.OUTAVG ... -0.225935
29PC0006:29PC0006.OUTAVG ... 0.446696
29PC0077:29PC0077.OUTAVG ... -0.216544
32LC0015:32LC0015.OUTAVG ... 0.352679
32PC1012:32PC1012.OUTAVG ... -0.116274
29PC0147:29PC0147.OUTAVG ... -0.208943
29PC0150:29PC0150.OUTAVG ... -0.214975
32LC0673:32LC0673.OUTAVG ... -0.199288
29FC0073:29FC0073.OUTAVG ... 0.225935
32FC0677:32FC0677.OUTAVG ... 1.000000
32TC0803:32TI0803.OUTAVG ... -0.217007
32LC0628:32LC0628.OUTAVG ... -0.527030
32LC0642:32LC0642.OUTAVG ... 0.257922
32DC0679:32DC0679.OUTAVG ... 0.942962
32DC0670:32DC0670.OUTAVG ... 0.432571
32FC0669:32FC0669.OUTAVG ... 0.710613
32PC0093:32PC0093.OUTAVG ... -0.161081
32LC0100:32LC0100.OUTAVG ... 0.320981
32TC0081:32TC0081.OUTAVG ... -0.091861
32TC0803:32TI0803.OUTAVG 32LC0628:32LC0628.OUTAVG \
32PICR03:32PACR03.PNT 0.158009 0.254340
32LC0119:32LC0119.OUTAVG 0.137659 0.660501
32LC0274:32LC0274.OUTAVG 0.269376 0.618263
32NC0273:32NE0273.OUT 0.006274 -0.110114
32PC0272:32PC0272.OUTAVG -0.291353 -0.845388
32DC0622:32DC0622.OUTAVG -0.286781 -0.822914
32LC0624:32LC0624.OUTAVG -0.017792 -0.402491
32LC0636:32LC0636.OUTAVG 0.049717 0.270156
32LC0682:32LC0682.OUTAVG 0.043038 -0.146512
32PC0611:32PC0611.OUTAVG -0.164714 -0.279592
32PC0623:32PC0623.OUTAVG 0.221910 0.186990
32PC0635:32PC0635.OUTAVG 0.332719 0.560519
32PC0676:32PC0676.OUTAVG 0.311824 0.486717
32PC0681:32PC0681.OUTAVG 0.263590 0.375569
32PC0905:32PC0905.OUTAVG -0.450835 -0.781729
32PI13AG:32PA13AG.PNT 0.139993 0.218453
32PC0153:32PC0153.OUTAVG 0.327502 0.463274
32TC15AR:32TC15AR.OUTAVG 0.287880 0.151896
32TC16AR:32TC16AR.OUTAVG 0.290377 -0.159424
32LC0619:32LC0619.OUTAVG 0.328099 0.575375
29FC0074:29FC0074.OUTAVG 0.383242 0.857668
29FC0334:29FC0334.OUTAVG -0.284635 -0.656995
29FCA093:29FC0093.OUTAVG 0.477694 0.856151
29HC0092:29HC0092.OUTAVG 0.265852 0.343033
29LC0071:29LC0071.OUTAVG 0.177110 0.373317
29LC0088:29LC0088.OUTAVG 0.303170 0.408433
29PC0006:29PC0006.OUTAVG -0.173560 -0.418486
29PC0077:29PC0077.OUTAVG -0.205795 0.424059
32LC0015:32LC0015.OUTAVG -0.227899 -0.399835
32PC1012:32PC1012.OUTAVG -0.253115 -0.297274
29PC0147:29PC0147.OUTAVG -0.098341 0.518704
29PC0150:29PC0150.OUTAVG 0.151348 0.454511
32LC0673:32LC0673.OUTAVG 0.296569 0.414468
29FC0073:29FC0073.OUTAVG -0.353618 -0.691062
32FC0677:32FC0677.OUTAVG -0.217007 -0.527030
32TC0803:32TI0803.OUTAVG 1.000000 0.419873
32LC0628:32LC0628.OUTAVG 0.419873 1.000000
32LC0642:32LC0642.OUTAVG 0.012065 0.001489
32DC0679:32DC0679.OUTAVG -0.083640 -0.244083
32DC0670:32DC0670.OUTAVG 0.005180 0.222090
32FC0669:32FC0669.OUTAVG -0.259267 -0.327651
32PC0093:32PC0093.OUTAVG 0.392483 0.400762
32LC0100:32LC0100.OUTAVG -0.411306 -0.823998
32TC0081:32TC0081.OUTAVG 0.014507 0.585097
32LC0642:32LC0642.OUTAVG 32DC0679:32DC0679.OUTAVG \
32PICR03:32PACR03.PNT -0.003988 -0.034241
32LC0119:32LC0119.OUTAVG 0.051220 0.005043
32LC0274:32LC0274.OUTAVG 0.093255 0.105932
32NC0273:32NE0273.OUT 0.048937 -0.065931
32PC0272:32PC0272.OUTAVG -0.083607 -0.094842
32DC0622:32DC0622.OUTAVG -0.350335 0.086085
32LC0624:32LC0624.OUTAVG 0.370945 -0.036391
32LC0636:32LC0636.OUTAVG 0.162552 0.191690
32LC0682:32LC0682.OUTAVG 0.231706 0.022625
32PC0611:32PC0611.OUTAVG -0.229093 -0.243294
32PC0623:32PC0623.OUTAVG 0.398329 -0.324226
32PC0635:32PC0635.OUTAVG 0.425943 -0.262904
32PC0676:32PC0676.OUTAVG 0.408714 -0.348141
32PC0681:32PC0681.OUTAVG 0.312081 -0.554629
32PC0905:32PC0905.OUTAVG -0.027021 -0.036308
32PI13AG:32PA13AG.PNT -0.059016 -0.062362
32PC0153:32PC0153.OUTAVG -0.000178 -0.002753
32TC15AR:32TC15AR.OUTAVG 0.483755 -0.029753
32TC16AR:32TC16AR.OUTAVG 0.187035 -0.141929
32LC0619:32LC0619.OUTAVG 0.364645 -0.209035
29FC0074:29FC0074.OUTAVG 0.039027 -0.046154
29FC0334:29FC0334.OUTAVG -0.165167 -0.093117
29FCA093:29FC0093.OUTAVG 0.290847 -0.173245
29HC0092:29HC0092.OUTAVG 0.592466 -0.014864
29LC0071:29LC0071.OUTAVG 0.419055 0.059735
29LC0088:29LC0088.OUTAVG 0.510193 -0.201017
29PC0006:29PC0006.OUTAVG 0.118277 0.366314
29PC0077:29PC0077.OUTAVG -0.131925 -0.074860
32LC0015:32LC0015.OUTAVG 0.287399 0.259119
32PC1012:32PC1012.OUTAVG -0.196466 -0.203571
29PC0147:29PC0147.OUTAVG 0.032957 -0.052386
29PC0150:29PC0150.OUTAVG -0.271511 -0.057612
32LC0673:32LC0673.OUTAVG -0.178149 -0.039685
29FC0073:29FC0073.OUTAVG -0.135210 -0.035491
32FC0677:32FC0677.OUTAVG 0.257922 0.942962
32TC0803:32TI0803.OUTAVG 0.012065 -0.083640
32LC0628:32LC0628.OUTAVG 0.001489 -0.244083
32LC0642:32LC0642.OUTAVG 1.000000 0.261286
32DC0679:32DC0679.OUTAVG 0.261286 1.000000
32DC0670:32DC0670.OUTAVG 0.222945 0.621311
32FC0669:32FC0669.OUTAVG 0.273351 0.708255
32PC0093:32PC0093.OUTAVG -0.029958 -0.015902
32LC0100:32LC0100.OUTAVG -0.172875 0.076047
32TC0081:32TC0081.OUTAVG 0.139793 0.106489
32DC0670:32DC0670.OUTAVG 32FC0669:32FC0669.OUTAVG \
32PICR03:32PACR03.PNT 0.219192 0.047379
32LC0119:32LC0119.OUTAVG 0.337535 -0.046931
32LC0274:32LC0274.OUTAVG 0.369496 -0.000065
32NC0273:32NE0273.OUT -0.092657 -0.017864
32PC0272:32PC0272.OUTAVG -0.535680 0.000083
32DC0622:32DC0622.OUTAVG -0.240620 0.154438
32LC0624:32LC0624.OUTAVG -0.314549 0.029008
32LC0636:32LC0636.OUTAVG 0.312175 0.207125
32LC0682:32LC0682.OUTAVG -0.213730 -0.002653
32PC0611:32PC0611.OUTAVG -0.432285 -0.213937
32PC0623:32PC0623.OUTAVG -0.401361 -0.349251
32PC0635:32PC0635.OUTAVG -0.120975 -0.292464
32PC0676:32PC0676.OUTAVG -0.268483 -0.398288
32PC0681:32PC0681.OUTAVG -0.357416 -0.439344
32PC0905:32PC0905.OUTAVG -0.496440 0.012589
32PI13AG:32PA13AG.PNT 0.081057 -0.077274
32PC0153:32PC0153.OUTAVG 0.252595 -0.029325
32TC15AR:32TC15AR.OUTAVG -0.025824 -0.014559
32TC16AR:32TC16AR.OUTAVG -0.235034 -0.084315
32LC0619:32LC0619.OUTAVG 0.047126 -0.151777
29FC0074:29FC0074.OUTAVG 0.401669 -0.125142
29FC0334:29FC0334.OUTAVG -0.438893 -0.060127
29FCA093:29FC0093.OUTAVG 0.172129 -0.260716
29HC0092:29HC0092.OUTAVG 0.026487 -0.038514
29LC0071:29LC0071.OUTAVG 0.144508 0.024537
29LC0088:29LC0088.OUTAVG -0.145566 -0.234066
29PC0006:29PC0006.OUTAVG 0.133765 0.349081
29PC0077:29PC0077.OUTAVG 0.203840 -0.056040
32LC0015:32LC0015.OUTAVG 0.060765 0.279651
32PC1012:32PC1012.OUTAVG -0.246463 -0.131767
29PC0147:29PC0147.OUTAVG 0.215592 -0.068514
29PC0150:29PC0150.OUTAVG 0.075603 -0.223989
32LC0673:32LC0673.OUTAVG 0.330954 0.053693
29FC0073:29FC0073.OUTAVG -0.430332 0.021134
32FC0677:32FC0677.OUTAVG 0.432571 0.710613
32TC0803:32TI0803.OUTAVG 0.005180 -0.259267
32LC0628:32LC0628.OUTAVG 0.222090 -0.327651
32LC0642:32LC0642.OUTAVG 0.222945 0.273351
32DC0679:32DC0679.OUTAVG 0.621311 0.708255
32DC0670:32DC0670.OUTAVG 1.000000 0.820939
32FC0669:32FC0669.OUTAVG 0.820939 1.000000
32PC0093:32PC0093.OUTAVG 0.149767 -0.114462
32LC0100:32LC0100.OUTAVG -0.331227 0.113286
32TC0081:32TC0081.OUTAVG 0.321153 -0.010677
32PC0093:32PC0093.OUTAVG 32LC0100:32LC0100.OUTAVG \
32PICR03:32PACR03.PNT 0.054669 -0.216792
32LC0119:32LC0119.OUTAVG 0.289715 -0.573117
32LC0274:32LC0274.OUTAVG 0.438569 -0.709061
32NC0273:32NE0273.OUT -0.102805 0.124166
32PC0272:32PC0272.OUTAVG -0.447188 0.789489
32DC0622:32DC0622.OUTAVG -0.261902 0.773437
32LC0624:32LC0624.OUTAVG -0.208077 0.263969
32LC0636:32LC0636.OUTAVG -0.024312 -0.349889
32LC0682:32LC0682.OUTAVG -0.158080 0.049758
32PC0611:32PC0611.OUTAVG -0.339803 0.407172
32PC0623:32PC0623.OUTAVG 0.105896 -0.230349
32PC0635:32PC0635.OUTAVG 0.243408 -0.585431
32PC0676:32PC0676.OUTAVG 0.201445 -0.495743
32PC0681:32PC0681.OUTAVG 0.107091 -0.377545
32PC0905:32PC0905.OUTAVG -0.442902 0.689014
32PI13AG:32PA13AG.PNT 0.065711 -0.134184
32PC0153:32PC0153.OUTAVG 0.578992 -0.601748
32TC15AR:32TC15AR.OUTAVG 0.084068 -0.297542
32TC16AR:32TC16AR.OUTAVG -0.005030 -0.001457
32LC0619:32LC0619.OUTAVG 0.182158 -0.569038
29FC0074:29FC0074.OUTAVG 0.383306 -0.711895
29FC0334:29FC0334.OUTAVG -0.315120 0.646214
29FCA093:29FC0093.OUTAVG 0.404959 -0.824607
29HC0092:29HC0092.OUTAVG 0.179184 -0.481444
29LC0071:29LC0071.OUTAVG 0.054023 -0.420882
29LC0088:29LC0088.OUTAVG 0.254896 -0.511115
29PC0006:29PC0006.OUTAVG -0.194417 0.287633
29PC0077:29PC0077.OUTAVG -0.099689 -0.129994
32LC0015:32LC0015.OUTAVG -0.159837 0.237873
32PC1012:32PC1012.OUTAVG -0.389616 0.469497
29PC0147:29PC0147.OUTAVG 0.010708 -0.275706
29PC0150:29PC0150.OUTAVG 0.169073 -0.185121
32LC0673:32LC0673.OUTAVG 0.245039 -0.294364
29FC0073:29FC0073.OUTAVG -0.395298 0.658962
32FC0677:32FC0677.OUTAVG -0.161081 0.320981
32TC0803:32TI0803.OUTAVG 0.392483 -0.411306
32LC0628:32LC0628.OUTAVG 0.400762 -0.823998
32LC0642:32LC0642.OUTAVG -0.029958 -0.172875
32DC0679:32DC0679.OUTAVG -0.015902 0.076047
32DC0670:32DC0670.OUTAVG 0.149767 -0.331227
32FC0669:32FC0669.OUTAVG -0.114462 0.113286
32PC0093:32PC0093.OUTAVG 1.000000 -0.572037
32LC0100:32LC0100.OUTAVG -0.572037 1.000000
32TC0081:32TC0081.OUTAVG 0.282009 -0.517215
32TC0081:32TC0081.OUTAVG
32PICR03:32PACR03.PNT -0.172643
32LC0119:32LC0119.OUTAVG 0.554931
32LC0274:32LC0274.OUTAVG 0.451645
32NC0273:32NE0273.OUT -0.090337
32PC0272:32PC0272.OUTAVG -0.600262
32DC0622:32DC0622.OUTAVG -0.647195
32LC0624:32LC0624.OUTAVG -0.315244
32LC0636:32LC0636.OUTAVG 0.358427
32LC0682:32LC0682.OUTAVG 0.054969
32PC0611:32PC0611.OUTAVG -0.220934
32PC0623:32PC0623.OUTAVG 0.062196
32PC0635:32PC0635.OUTAVG 0.371786
32PC0676:32PC0676.OUTAVG 0.335426
32PC0681:32PC0681.OUTAVG 0.079680
32PC0905:32PC0905.OUTAVG -0.388246
32PI13AG:32PA13AG.PNT 0.057669
32PC0153:32PC0153.OUTAVG 0.243496
32TC15AR:32TC15AR.OUTAVG 0.031255
32TC16AR:32TC16AR.OUTAVG -0.476943
32LC0619:32LC0619.OUTAVG 0.410766
29FC0074:29FC0074.OUTAVG 0.574426
29FC0334:29FC0334.OUTAVG -0.491838
29FCA093:29FC0093.OUTAVG 0.535344
29HC0092:29HC0092.OUTAVG 0.294031
29LC0071:29LC0071.OUTAVG 0.202209
29LC0088:29LC0088.OUTAVG 0.282517
29PC0006:29PC0006.OUTAVG -0.019024
29PC0077:29PC0077.OUTAVG 0.542126
32LC0015:32LC0015.OUTAVG -0.319161
32PC1012:32PC1012.OUTAVG -0.289113
29PC0147:29PC0147.OUTAVG 0.670041
29PC0150:29PC0150.OUTAVG 0.406225
32LC0673:32LC0673.OUTAVG 0.223854
29FC0073:29FC0073.OUTAVG -0.527557
32FC0677:32FC0677.OUTAVG -0.091861
32TC0803:32TI0803.OUTAVG 0.014507
32LC0628:32LC0628.OUTAVG 0.585097
32LC0642:32LC0642.OUTAVG 0.139793
32DC0679:32DC0679.OUTAVG 0.106489
32DC0670:32DC0670.OUTAVG 0.321153
32FC0669:32FC0669.OUTAVG -0.010677
32PC0093:32PC0093.OUTAVG 0.282009
32LC0100:32LC0100.OUTAVG -0.517215
32TC0081:32TC0081.OUTAVG 1.000000
[44 rows x 44 columns]
Filled missing values in 32PICR03:32PACR03.PNT using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32PICR03:32PACR03.PNT', '32LC0119:32LC0119.OUTAVG', '32LC0274:32LC0274.OUTAVG', '32NC0273:32NE0273.OUT', '32PC0272:32PC0272.OUTAVG', '32DC0622:32DC0622.OUTAVG', '32LC0624:32LC0624.OUTAVG', '32LC0636:32LC0636.OUTAVG', '32LC0682:32LC0682.OUTAVG', '32PC0611:32PC0611.OUTAVG', '32PC0623:32PC0623.OUTAVG', '32PC0635:32PC0635.OUTAVG', '32PC0676:32PC0676.OUTAVG', '32PC0681:32PC0681.OUTAVG', '32PC0905:32PC0905.OUTAVG', '32PI13AG:32PA13AG.PNT', '32PC0153:32PC0153.OUTAVG', '32TC15AR:32TC15AR.OUTAVG', '32TC16AR:32TC16AR.OUTAVG', '32LC0619:32LC0619.OUTAVG', '29FC0074:29FC0074.OUTAVG', '29FC0334:29FC0334.OUTAVG', '29FCA093:29FC0093.OUTAVG', '29HC0092:29HC0092.OUTAVG', '29LC0071:29LC0071.OUTAVG', '29LC0088:29LC0088.OUTAVG', '29PC0006:29PC0006.OUTAVG', '29PC0077:29PC0077.OUTAVG', '32LC0015:32LC0015.OUTAVG', '32PC1012:32PC1012.OUTAVG', '29PC0147:29PC0147.OUTAVG', '29PC0150:29PC0150.OUTAVG', '32LC0673:32LC0673.OUTAVG', '29FC0073:29FC0073.OUTAVG', '32FC0677:32FC0677.OUTAVG', '32TC0803:32TI0803.OUTAVG', '32LC0628:32LC0628.OUTAVG', '32LC0642:32LC0642.OUTAVG', '32DC0679:32DC0679.OUTAVG', '32DC0670:32DC0670.OUTAVG', '32FC0669:32FC0669.OUTAVG', '32PC0093:32PC0093.OUTAVG', '32LC0100:32LC0100.OUTAVG', '32TC0081:32TC0081.OUTAVG']
Filled missing values in 32PICR03:32PACR03.PNT using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Average:
32RLBWTLSA 32RLMOILSA
32RLBWTLSA 1.000000 0.232899
32RLMOILSA 0.232899 1.000000
Filled missing values in medium correlation columns using MICE: ['32RLBWTLSA', '32RLMOILSA']
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Refiner:
29JC0023:29JD0023.RI07 29JC0023:29JD0023.RI08 \
29JC0023:29JD0023.RI07 1.000000 0.160469
29JC0023:29JD0023.RI08 0.160469 1.000000
29JC0023:29JD0023.RO01 0.958680 0.030636
29JC0023:29JD0023.RO02 0.088250 0.768045
29JC133A:29JA133A.PNT -0.168494 0.246133
29JC133A:29JG133A.RI06 0.005635 0.471848
29JC133A:29JG133A.RO03 -0.073525 0.136747
29JC0023:29JD0023.RO01 29JC0023:29JD0023.RO02 \
29JC0023:29JD0023.RI07 0.958680 0.088250
29JC0023:29JD0023.RI08 0.030636 0.768045
29JC0023:29JD0023.RO01 1.000000 0.053578
29JC0023:29JD0023.RO02 0.053578 1.000000
29JC133A:29JA133A.PNT -0.191912 0.206417
29JC133A:29JG133A.RI06 -0.062678 0.313710
29JC133A:29JG133A.RO03 -0.067878 0.147699
29JC133A:29JA133A.PNT 29JC133A:29JG133A.RI06 \
29JC0023:29JD0023.RI07 -0.168494 0.005635
29JC0023:29JD0023.RI08 0.246133 0.471848
29JC0023:29JD0023.RO01 -0.191912 -0.062678
29JC0023:29JD0023.RO02 0.206417 0.313710
29JC133A:29JA133A.PNT 1.000000 0.339565
29JC133A:29JG133A.RI06 0.339565 1.000000
29JC133A:29JG133A.RO03 0.448490 0.855496
29JC133A:29JG133A.RO03
29JC0023:29JD0023.RI07 -0.073525
29JC0023:29JD0023.RI08 0.136747
29JC0023:29JD0023.RO01 -0.067878
29JC0023:29JD0023.RO02 0.147699
29JC133A:29JA133A.PNT 0.448490
29JC133A:29JG133A.RI06 0.855496
29JC133A:29JG133A.RO03 1.000000
Filled missing values in 29JC0023:29JD0023.RI07 using Random Forest.
Filled missing values in 29JC0023:29JD0023.RI08 using Random Forest.
Filled missing values in 29JC0023:29JD0023.RO01 using Random Forest.
Filled missing values in 29JC133A:29JA133A.PNT using Random Forest.
Filled missing values in 29JC133A:29JG133A.RI06 using Random Forest.
Filled missing values in 29JC133A:29JG133A.RO03 using Random Forest.
Filled missing values in medium correlation columns using MICE: ['29JC0023:29JD0023.RI07', '29JC0023:29JD0023.RI08', '29JC0023:29JD0023.RO01', '29JC0023:29JD0023.RO02', '29JC133A:29JA133A.PNT', '29JC133A:29JG133A.RI06', '29JC133A:29JG133A.RO03']
Filled missing values in 29JC0023:29JD0023.RI07 using forward and backward fill.
Filled missing values in 29JC0023:29JD0023.RI08 using forward and backward fill.
Filled missing values in 29JC0023:29JD0023.RO01 using forward and backward fill.
Filled missing values in 29JC133A:29JA133A.PNT using forward and backward fill.
Filled missing values in 29JC133A:29JG133A.RI06 using forward and backward fill.
Filled missing values in 29JC133A:29JG133A.RO03 using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Reel:
32RL1BWTCD 32RL1COLaACT 32RL1MOICD \
32RL1BWTCD 1.000000 0.038622 0.499732
32RL1COLaACT 0.038622 1.000000 0.037874
32RL1MOICD 0.499732 0.037874 1.000000
32W2.PMREEL_IOP -0.068042 0.012199 -0.024793
HMX_WRITE:HR40044.PNT 0.183345 0.039039 -0.000026
32RLPRODWT -0.116085 -0.152666 -0.196037
32DRVMST_RD6:REELDRUM_DRW.MEAS -0.148508 -0.014137 0.068487
32DRVMST_RD6:REELDRUM_CUR.MEAS 0.041375 -0.032316 0.127011
32W2.PMREEL_IOP HMX_WRITE:HR40044.PNT \
32RL1BWTCD -0.068042 0.183345
32RL1COLaACT 0.012199 0.039039
32RL1MOICD -0.024793 -0.000026
32W2.PMREEL_IOP 1.000000 -0.032913
HMX_WRITE:HR40044.PNT -0.032913 1.000000
32RLPRODWT 0.097214 -0.046097
32DRVMST_RD6:REELDRUM_DRW.MEAS -0.004462 -0.341916
32DRVMST_RD6:REELDRUM_CUR.MEAS 0.028317 0.005103
32RLPRODWT 32DRVMST_RD6:REELDRUM_DRW.MEAS \
32RL1BWTCD -0.116085 -0.148508
32RL1COLaACT -0.152666 -0.014137
32RL1MOICD -0.196037 0.068487
32W2.PMREEL_IOP 0.097214 -0.004462
HMX_WRITE:HR40044.PNT -0.046097 -0.341916
32RLPRODWT 1.000000 0.017125
32DRVMST_RD6:REELDRUM_DRW.MEAS 0.017125 1.000000
32DRVMST_RD6:REELDRUM_CUR.MEAS 0.028634 0.065348
32DRVMST_RD6:REELDRUM_CUR.MEAS
32RL1BWTCD 0.041375
32RL1COLaACT -0.032316
32RL1MOICD 0.127011
32W2.PMREEL_IOP 0.028317
HMX_WRITE:HR40044.PNT 0.005103
32RLPRODWT 0.028634
32DRVMST_RD6:REELDRUM_DRW.MEAS 0.065348
32DRVMST_RD6:REELDRUM_CUR.MEAS 1.000000
Filled missing values in 32RL1BWTCD using Random Forest.
Filled missing values in 32RL1COLaACT using Random Forest.
Filled missing values in 32RL1MOICD using Random Forest.
Filled missing values in 32W2.PMREEL_IOP using Random Forest.
Filled missing values in HMX_WRITE:HR40044.PNT using Random Forest.
Filled missing values in 32RLPRODWT using Random Forest.
Filled missing values in 32DRVMST_RD6:REELDRUM_DRW.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32RL1BWTCD', '32RL1COLaACT', '32RL1MOICD', '32W2.PMREEL_IOP', 'HMX_WRITE:HR40044.PNT', '32RLPRODWT', '32DRVMST_RD6:REELDRUM_DRW.MEAS', '32DRVMST_RD6:REELDRUM_CUR.MEAS']
Filled missing values in 32RL1BWTCD using forward and backward fill.
Filled missing values in 32RL1COLaACT using forward and backward fill.
Filled missing values in 32RL1MOICD using forward and backward fill.
Filled missing values in 32W2.PMREEL_IOP using forward and backward fill.
Filled missing values in HMX_WRITE:HR40044.PNT using forward and backward fill.
Filled missing values in 32RLPRODWT using forward and backward fill.
Filled missing values in 32DRVMST_RD6:REELDRUM_DRW.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Roll:
32PC0145:32PC0145.MEAS 32PI20AP:32PA20AP.PNT \
32PC0145:32PC0145.MEAS 1.000000 -0.039587
32PI20AP:32PA20AP.PNT -0.039587 1.000000
32PI20AQ:32PA20AQ.PNT 0.106991 0.165075
32PI82AL:32PA82AL.PNT -0.042294 0.137066
32PI20AQ:32PA20AQ.PNT 32PI82AL:32PA82AL.PNT
32PC0145:32PC0145.MEAS 0.106991 -0.042294
32PI20AP:32PA20AP.PNT 0.165075 0.137066
32PI20AQ:32PA20AQ.PNT 1.000000 0.040998
32PI82AL:32PA82AL.PNT 0.040998 1.000000
Filled missing values in 32PI20AP:32PA20AP.PNT using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32PC0145:32PC0145.MEAS', '32PI20AP:32PA20AP.PNT', '32PI20AQ:32PA20AQ.PNT', '32PI82AL:32PA82AL.PNT']
Filled missing values in 32PI20AP:32PA20AP.PNT using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Weight:
29FC0320:29FA0320.RO02 29FC0360:29FD0360.RO02 \
29FC0320:29FA0320.RO02 1.000000 0.323139
29FC0360:29FD0360.RO02 0.323139 1.000000
32FC1013:32FD1013.RO01 0.042353 0.505339
32G560:560CALC.RO03 -0.013428 0.097980
ASA_SIZE:CALC_PM2.RO01 0.019404 -0.037587
32G482:SILICA_CALC.RO02 NaN NaN
32FC1013:32FD1013.RO01 32G560:560CALC.RO03 \
29FC0320:29FA0320.RO02 0.042353 -0.013428
29FC0360:29FD0360.RO02 0.505339 0.097980
32FC1013:32FD1013.RO01 1.000000 0.076765
32G560:560CALC.RO03 0.076765 1.000000
ASA_SIZE:CALC_PM2.RO01 -0.013659 0.006548
32G482:SILICA_CALC.RO02 NaN NaN
ASA_SIZE:CALC_PM2.RO01 32G482:SILICA_CALC.RO02
29FC0320:29FA0320.RO02 0.019404 NaN
29FC0360:29FD0360.RO02 -0.037587 NaN
32FC1013:32FD1013.RO01 -0.013659 NaN
32G560:560CALC.RO03 0.006548 NaN
ASA_SIZE:CALC_PM2.RO01 1.000000 NaN
32G482:SILICA_CALC.RO02 NaN NaN
Length mismatch when trying to fill 32G560:560CALC.RO03. Skipping this column.
Length mismatch when trying to fill 32G482:SILICA_CALC.RO02. Skipping this column.
Filled missing values in medium correlation columns using MICE: ['29FC0320:29FA0320.RO02', '29FC0360:29FD0360.RO02', '32FC1013:32FD1013.RO01', '32G560:560CALC.RO03', 'ASA_SIZE:CALC_PM2.RO01', '32G482:SILICA_CALC.RO02']
Filled missing values in 32G560:560CALC.RO03 using forward and backward fill.
Filled missing values in 32G482:SILICA_CALC.RO02 using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Pump:
32HC0080:32HA0080.OUT 32II0168E:32IA0168E.PNT \
32HC0080:32HA0080.OUT 1.000000 0.014199
32II0168E:32IA0168E.PNT 0.014199 1.000000
29II0001A:32IA454A.PNT 0.264472 -0.045835
29II0001A:32IA453A.PNT -0.225755 0.048381
18NC0833:18NA0833.PNT 0.042855 -0.019126
29II0001A:32IA454A.PNT 29II0001A:32IA453A.PNT \
32HC0080:32HA0080.OUT 0.264472 -0.225755
32II0168E:32IA0168E.PNT -0.045835 0.048381
29II0001A:32IA454A.PNT 1.000000 -0.987145
29II0001A:32IA453A.PNT -0.987145 1.000000
18NC0833:18NA0833.PNT 0.089609 -0.095344
18NC0833:18NA0833.PNT
32HC0080:32HA0080.OUT 0.042855
32II0168E:32IA0168E.PNT -0.019126
29II0001A:32IA454A.PNT 0.089609
29II0001A:32IA453A.PNT -0.095344
18NC0833:18NA0833.PNT 1.000000
Filled missing values in 32HC0080:32HA0080.OUT using Random Forest.
Filled missing values in 29II0001A:32IA454A.PNT using Random Forest.
Filled missing values in 29II0001A:32IA453A.PNT using Random Forest.
Filled missing values in 18NC0833:18NA0833.PNT using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32HC0080:32HA0080.OUT', '32II0168E:32IA0168E.PNT', '29II0001A:32IA454A.PNT', '29II0001A:32IA453A.PNT', '18NC0833:18NA0833.PNT']
Filled missing values in 32HC0080:32HA0080.OUT using forward and backward fill.
Filled missing values in 29II0001A:32IA454A.PNT using forward and backward fill.
Filled missing values in 29II0001A:32IA453A.PNT using forward and backward fill.
Filled missing values in 18NC0833:18NA0833.PNT using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Box:
29FCA093:29FA0093.PNT 32FI0086:32FA0086.PNT \
29FCA093:29FA0093.PNT 1.000000 0.867906
32FI0086:32FA0086.PNT 0.867906 1.000000
32HC0090:32HA0090.OUT -0.522512 -0.184898
32PC0160:32PC0160.MEAS 0.055597 -0.048498
32PI0176:32PA0176.PNT -0.328611 -0.341329
32PI0177:32PA0177.PNT 0.054485 0.002268
32PI0178:32PA0178.PNT -0.035332 -0.015594
32ZC0346:32ZA0346.PNT -0.723432 -0.498145
29FCA093:29HC0094.OUT 0.994867 0.847169
32HC0090:32HA0090.OUT 32PC0160:32PC0160.MEAS \
29FCA093:29FA0093.PNT -0.522512 0.055597
32FI0086:32FA0086.PNT -0.184898 -0.048498
32HC0090:32HA0090.OUT 1.000000 -0.069493
32PC0160:32PC0160.MEAS -0.069493 1.000000
32PI0176:32PA0176.PNT -0.026711 -0.158385
32PI0177:32PA0177.PNT -0.098999 -0.102422
32PI0178:32PA0178.PNT -0.090892 -0.050433
32ZC0346:32ZA0346.PNT 0.526002 -0.080567
29FCA093:29HC0094.OUT -0.510152 0.047076
32PI0176:32PA0176.PNT 32PI0177:32PA0177.PNT \
29FCA093:29FA0093.PNT -0.328611 0.054485
32FI0086:32FA0086.PNT -0.341329 0.002268
32HC0090:32HA0090.OUT -0.026711 -0.098999
32PC0160:32PC0160.MEAS -0.158385 -0.102422
32PI0176:32PA0176.PNT 1.000000 0.244054
32PI0177:32PA0177.PNT 0.244054 1.000000
32PI0178:32PA0178.PNT 0.408251 -0.119848
32ZC0346:32ZA0346.PNT 0.263575 -0.010610
29FCA093:29HC0094.OUT -0.332783 0.060577
32PI0178:32PA0178.PNT 32ZC0346:32ZA0346.PNT \
29FCA093:29FA0093.PNT -0.035332 -0.723432
32FI0086:32FA0086.PNT -0.015594 -0.498145
32HC0090:32HA0090.OUT -0.090892 0.526002
32PC0160:32PC0160.MEAS -0.050433 -0.080567
32PI0176:32PA0176.PNT 0.408251 0.263575
32PI0177:32PA0177.PNT -0.119848 -0.010610
32PI0178:32PA0178.PNT 1.000000 -0.076708
32ZC0346:32ZA0346.PNT -0.076708 1.000000
29FCA093:29HC0094.OUT -0.047631 -0.737395
29FCA093:29HC0094.OUT
29FCA093:29FA0093.PNT 0.994867
32FI0086:32FA0086.PNT 0.847169
32HC0090:32HA0090.OUT -0.510152
32PC0160:32PC0160.MEAS 0.047076
32PI0176:32PA0176.PNT -0.332783
32PI0177:32PA0177.PNT 0.060577
32PI0178:32PA0178.PNT -0.047631
32ZC0346:32ZA0346.PNT -0.737395
29FCA093:29HC0094.OUT 1.000000
Filled missing values in 32HC0090:32HA0090.OUT using Random Forest.
Filled missing values in 32PC0160:32PC0160.MEAS using Random Forest.
Filled missing values in 32ZC0346:32ZA0346.PNT using Random Forest.
Filled missing values in medium correlation columns using MICE: ['29FCA093:29FA0093.PNT', '32FI0086:32FA0086.PNT', '32HC0090:32HA0090.OUT', '32PC0160:32PC0160.MEAS', '32PI0176:32PA0176.PNT', '32PI0177:32PA0177.PNT', '32PI0178:32PA0178.PNT', '32ZC0346:32ZA0346.PNT', '29FCA093:29HC0094.OUT']
Filled missing values in 32HC0090:32HA0090.OUT using forward and backward fill.
Filled missing values in 32PC0160:32PC0160.MEAS using forward and backward fill.
Filled missing values in 32ZC0346:32ZA0346.PNT using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Screen:
32DI0054:32DA0054.PNT 32II0017A:32IA0017A.PNT \
32DI0054:32DA0054.PNT 1.000000 0.089227
32II0017A:32IA0017A.PNT 0.089227 1.000000
28NC0008:28NC0008.MEAS -0.060829 -0.022370
28NC0008:28NC0008.MEAS
32DI0054:32DA0054.PNT -0.060829
32II0017A:32IA0017A.PNT -0.022370
28NC0008:28NC0008.MEAS 1.000000
Filled missing values in 28NC0008:28NC0008.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32DI0054:32DA0054.PNT', '32II0017A:32IA0017A.PNT', '28NC0008:28NC0008.MEAS']
Filled missing values in 28NC0008:28NC0008.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Receiver:
32II0003A:32IA0003A.PNT 32II0413A:32IA0413A.PNT \
32II0003A:32IA0003A.PNT 1.000000 0.419354
32II0413A:32IA0413A.PNT 0.419354 1.000000
32NI0077:32NA0077.PNT 0.451909 0.208546
32PI0851:32PA0851.PNT 0.029164 0.074159
32PI0853:32PA0853.PNT 0.020739 -0.225210
32PI0855:32PA0855.PNT 0.003099 -0.286752
32TC0081:32TC0081.MEAS -0.094045 0.008437
32TC0081:32TC0081.OUT 0.146189 -0.081239
32LC0756:32LC0756.MEAS 0.386138 0.140241
32NI0077:32NC0077.MEAS 0.451729 0.208508
32NI0077:32NA0077.PNT 32PI0851:32PA0851.PNT \
32II0003A:32IA0003A.PNT 0.451909 0.029164
32II0413A:32IA0413A.PNT 0.208546 0.074159
32NI0077:32NA0077.PNT 1.000000 0.081836
32PI0851:32PA0851.PNT 0.081836 1.000000
32PI0853:32PA0853.PNT 0.181360 0.818314
32PI0855:32PA0855.PNT 0.193663 0.740396
32TC0081:32TC0081.MEAS -0.073160 -0.070899
32TC0081:32TC0081.OUT 0.167035 0.064279
32LC0756:32LC0756.MEAS 0.336083 0.074548
32NI0077:32NC0077.MEAS 0.999945 0.081605
32PI0853:32PA0853.PNT 32PI0855:32PA0855.PNT \
32II0003A:32IA0003A.PNT 0.020739 0.003099
32II0413A:32IA0413A.PNT -0.225210 -0.286752
32NI0077:32NA0077.PNT 0.181360 0.193663
32PI0851:32PA0851.PNT 0.818314 0.740396
32PI0853:32PA0853.PNT 1.000000 0.962288
32PI0855:32PA0855.PNT 0.962288 1.000000
32TC0081:32TC0081.MEAS -0.117587 -0.136534
32TC0081:32TC0081.OUT 0.220918 0.185831
32LC0756:32LC0756.MEAS 0.149257 0.177928
32NI0077:32NC0077.MEAS 0.181110 0.193306
32TC0081:32TC0081.MEAS 32TC0081:32TC0081.OUT \
32II0003A:32IA0003A.PNT -0.094045 0.146189
32II0413A:32IA0413A.PNT 0.008437 -0.081239
32NI0077:32NA0077.PNT -0.073160 0.167035
32PI0851:32PA0851.PNT -0.070899 0.064279
32PI0853:32PA0853.PNT -0.117587 0.220918
32PI0855:32PA0855.PNT -0.136534 0.185831
32TC0081:32TC0081.MEAS 1.000000 0.211665
32TC0081:32TC0081.OUT 0.211665 1.000000
32LC0756:32LC0756.MEAS -0.108746 0.520709
32NI0077:32NC0077.MEAS -0.073233 0.166988
32LC0756:32LC0756.MEAS 32NI0077:32NC0077.MEAS
32II0003A:32IA0003A.PNT 0.386138 0.451729
32II0413A:32IA0413A.PNT 0.140241 0.208508
32NI0077:32NA0077.PNT 0.336083 0.999945
32PI0851:32PA0851.PNT 0.074548 0.081605
32PI0853:32PA0853.PNT 0.149257 0.181110
32PI0855:32PA0855.PNT 0.177928 0.193306
32TC0081:32TC0081.MEAS -0.108746 -0.073233
32TC0081:32TC0081.OUT 0.520709 0.166988
32LC0756:32LC0756.MEAS 1.000000 0.335966
32NI0077:32NC0077.MEAS 0.335966 1.000000
Filled missing values in 32TC0081:32TC0081.MEAS using Random Forest.
Filled missing values in 32TC0081:32TC0081.OUT using Random Forest.
Filled missing values in 32LC0756:32LC0756.MEAS using Random Forest.
Filled missing values in 32NI0077:32NC0077.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32II0003A:32IA0003A.PNT', '32II0413A:32IA0413A.PNT', '32NI0077:32NA0077.PNT', '32PI0851:32PA0851.PNT', '32PI0853:32PA0853.PNT', '32PI0855:32PA0855.PNT', '32TC0081:32TC0081.MEAS', '32TC0081:32TC0081.OUT', '32LC0756:32LC0756.MEAS', '32NI0077:32NC0077.MEAS']
Filled missing values in 32TC0081:32TC0081.MEAS using forward and backward fill.
Filled missing values in 32TC0081:32TC0081.OUT using forward and backward fill.
Filled missing values in 32LC0756:32LC0756.MEAS using forward and backward fill.
Filled missing values in 32NI0077:32NC0077.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Aid:
29FC0186:29FC0186.MEAS 29FC0186:29FC0186.OUT \
29FC0186:29FC0186.MEAS 1.000000 0.060208
29FC0186:29FC0186.OUT 0.060208 1.000000
29POLYMER:29POLYMER.PNT 0.001419 0.022206
29POLYMER:29POLYMER.PNT
29FC0186:29FC0186.MEAS 0.001419
29FC0186:29FC0186.OUT 0.022206
29POLYMER:29POLYMER.PNT 1.000000
Filled missing values in 29FC0186:29FC0186.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['29FC0186:29FC0186.MEAS', '29FC0186:29FC0186.OUT', '29POLYMER:29POLYMER.PNT']
Filled missing values in 29FC0186:29FC0186.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Shower:
32PI0213:32PC0213.MEAS 32PI0801:32PA0801.PNT \
32PI0213:32PC0213.MEAS 1.000000 0.048550
32PI0801:32PA0801.PNT 0.048550 1.000000
32PI0813:32PI0813.PNT 0.213053 -0.013495
32PI0813:32PI0813.PNT
32PI0213:32PC0213.MEAS 0.213053
32PI0801:32PA0801.PNT -0.013495
32PI0813:32PI0813.PNT 1.000000
Filled missing values in 32PI0213:32PC0213.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['32PI0213:32PC0213.MEAS', '32PI0801:32PA0801.PNT', '32PI0813:32PI0813.PNT']
Filled missing values in 32PI0213:32PC0213.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:10: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df_category.dropna(how='all', inplace=True)
Correlation Matrix with Missing Data for Other Sensors:
29FC0078:29FC0078.MEAS 29NC0007:29NC0007.MEAS \
29FC0078:29FC0078.MEAS 1.000000 -0.152793
29NC0007:29NC0007.MEAS -0.152793 1.000000
32NA1024:32NA1024_SV.MEAS 0.129479 0.021718
32PC0098:32PE0098.RO03 -0.022307 0.033324
32PI25AJ:32PA25AJ.PNT -0.048009 0.094865
32PI25AK:32PA25AK.PNT 0.113967 -0.051938
32PICR01:32PACR01.PNT 0.011127 -0.066913
32PICR02:32PACR02.PNT -0.019498 0.118990
32ZI0350:32ZA0350.PNT -0.036996 -0.079891
38FC0050:38FB0050.RO01 0.040810 0.003214
40_STEAM:1RVMTR_MW_AI.PNT 0.058067 0.009943
40HORNS:MW_TEST.RO01 0.034450 -0.019038
40JI0333:40JI0333.PNT 0.021331 0.000335
32PC0676:32PE0676.OUT -0.084784 0.044304
32G558:558TCV1.OUT -0.122730 0.010587
HMX_WRITE:HR40041.PNT 0.027260 -0.007062
HMX_WRITE:HR40012.PNT 0.028984 -0.006434
32G558:558TCV1.OUT.24hravg -0.174987 0.043894
LFStorageChestOutletFlow_CALC 0.020595 -0.140351
28FC1030:28FH1030.RI02 0.044723 -0.006907
40PC0063:40PC0063.MEAS 0.046351 0.066986
PM2_TotalStarch.CV -0.239505 0.023768
PM2_Rush/Drag.AF -0.129330 0.051947
32DRVMST_RD:COUCH_CUR.MEAS -0.075676 -0.019118
32DRVMST_RD5:CAL_DRW.MEAS 0.047845 0.018859
32NA1024:32NA1024_SV.MEAS \
29FC0078:29FC0078.MEAS 0.129479
29NC0007:29NC0007.MEAS 0.021718
32NA1024:32NA1024_SV.MEAS 1.000000
32PC0098:32PE0098.RO03 -0.064397
32PI25AJ:32PA25AJ.PNT -0.015643
32PI25AK:32PA25AK.PNT 0.001035
32PICR01:32PACR01.PNT 0.095434
32PICR02:32PACR02.PNT -0.068717
32ZI0350:32ZA0350.PNT -0.260106
38FC0050:38FB0050.RO01 0.042658
40_STEAM:1RVMTR_MW_AI.PNT 0.089184
40HORNS:MW_TEST.RO01 0.148875
40JI0333:40JI0333.PNT 0.151262
32PC0676:32PE0676.OUT 0.073240
32G558:558TCV1.OUT -0.170775
HMX_WRITE:HR40041.PNT -0.055669
HMX_WRITE:HR40012.PNT -0.050690
32G558:558TCV1.OUT.24hravg -0.166980
LFStorageChestOutletFlow_CALC 0.060482
28FC1030:28FH1030.RI02 0.016904
40PC0063:40PC0063.MEAS 0.054379
PM2_TotalStarch.CV -0.248180
PM2_Rush/Drag.AF -0.026357
32DRVMST_RD:COUCH_CUR.MEAS -0.152866
32DRVMST_RD5:CAL_DRW.MEAS 0.051575
32PC0098:32PE0098.RO03 32PI25AJ:32PA25AJ.PNT \
29FC0078:29FC0078.MEAS -0.022307 -0.048009
29NC0007:29NC0007.MEAS 0.033324 0.094865
32NA1024:32NA1024_SV.MEAS -0.064397 -0.015643
32PC0098:32PE0098.RO03 1.000000 0.006675
32PI25AJ:32PA25AJ.PNT 0.006675 1.000000
32PI25AK:32PA25AK.PNT 0.068902 0.395918
32PICR01:32PACR01.PNT -0.192168 0.023459
32PICR02:32PACR02.PNT 0.218950 0.237424
32ZI0350:32ZA0350.PNT -0.078188 -0.007695
38FC0050:38FB0050.RO01 0.058018 -0.072946
40_STEAM:1RVMTR_MW_AI.PNT 0.075813 -0.062446
40HORNS:MW_TEST.RO01 0.033735 -0.007136
40JI0333:40JI0333.PNT -0.052291 -0.048913
32PC0676:32PE0676.OUT -0.572605 -0.055715
32G558:558TCV1.OUT 0.273409 0.055606
HMX_WRITE:HR40041.PNT -0.110600 -0.103705
HMX_WRITE:HR40012.PNT -0.108339 -0.103143
32G558:558TCV1.OUT.24hravg 0.282895 0.131677
LFStorageChestOutletFlow_CALC -0.491778 -0.119379
28FC1030:28FH1030.RI02 0.374027 0.013151
40PC0063:40PC0063.MEAS 0.050872 0.091954
PM2_TotalStarch.CV 0.035276 -0.019165
PM2_Rush/Drag.AF 0.279991 -0.070628
32DRVMST_RD:COUCH_CUR.MEAS 0.343323 0.174564
32DRVMST_RD5:CAL_DRW.MEAS -0.284449 -0.145373
32PI25AK:32PA25AK.PNT 32PICR01:32PACR01.PNT \
29FC0078:29FC0078.MEAS 0.113967 0.011127
29NC0007:29NC0007.MEAS -0.051938 -0.066913
32NA1024:32NA1024_SV.MEAS 0.001035 0.095434
32PC0098:32PE0098.RO03 0.068902 -0.192168
32PI25AJ:32PA25AJ.PNT 0.395918 0.023459
32PI25AK:32PA25AK.PNT 1.000000 0.109176
32PICR01:32PACR01.PNT 0.109176 1.000000
32PICR02:32PACR02.PNT 0.073824 -0.015795
32ZI0350:32ZA0350.PNT -0.096968 -0.057384
38FC0050:38FB0050.RO01 -0.185568 -0.123246
40_STEAM:1RVMTR_MW_AI.PNT -0.127095 -0.112426
40HORNS:MW_TEST.RO01 0.199174 0.095961
40JI0333:40JI0333.PNT 0.133950 0.115744
32PC0676:32PE0676.OUT -0.271334 0.145427
32G558:558TCV1.OUT -0.241287 -0.018314
HMX_WRITE:HR40041.PNT -0.198879 -0.084164
HMX_WRITE:HR40012.PNT -0.190218 -0.081709
32G558:558TCV1.OUT.24hravg -0.193777 -0.027052
LFStorageChestOutletFlow_CALC -0.061119 0.183956
28FC1030:28FH1030.RI02 0.298846 -0.083389
40PC0063:40PC0063.MEAS 0.082727 -0.031148
PM2_TotalStarch.CV -0.405514 -0.021655
PM2_Rush/Drag.AF -0.316186 -0.011533
32DRVMST_RD:COUCH_CUR.MEAS -0.014858 -0.011921
32DRVMST_RD5:CAL_DRW.MEAS -0.389314 0.125589
32PICR02:32PACR02.PNT 32ZI0350:32ZA0350.PNT \
29FC0078:29FC0078.MEAS -0.019498 -0.036996
29NC0007:29NC0007.MEAS 0.118990 -0.079891
32NA1024:32NA1024_SV.MEAS -0.068717 -0.260106
32PC0098:32PE0098.RO03 0.218950 -0.078188
32PI25AJ:32PA25AJ.PNT 0.237424 -0.007695
32PI25AK:32PA25AK.PNT 0.073824 -0.096968
32PICR01:32PACR01.PNT -0.015795 -0.057384
32PICR02:32PACR02.PNT 1.000000 -0.066795
32ZI0350:32ZA0350.PNT -0.066795 1.000000
38FC0050:38FB0050.RO01 -0.014854 0.079250
40_STEAM:1RVMTR_MW_AI.PNT 0.067079 0.058341
40HORNS:MW_TEST.RO01 -0.028319 -0.133796
40JI0333:40JI0333.PNT -0.062516 -0.140166
32PC0676:32PE0676.OUT -0.207632 -0.058623
32G558:558TCV1.OUT 0.103015 0.023498
HMX_WRITE:HR40041.PNT -0.090444 0.049019
HMX_WRITE:HR40012.PNT -0.092799 0.040712
32G558:558TCV1.OUT.24hravg 0.106196 0.018780
LFStorageChestOutletFlow_CALC -0.186196 0.069739
28FC1030:28FH1030.RI02 0.078444 0.047281
40PC0063:40PC0063.MEAS 0.025130 -0.007913
PM2_TotalStarch.CV 0.007491 0.141518
PM2_Rush/Drag.AF -0.039371 -0.056810
32DRVMST_RD:COUCH_CUR.MEAS 0.265767 0.093739
32DRVMST_RD5:CAL_DRW.MEAS -0.165420 -0.041489
38FC0050:38FB0050.RO01 ... \
29FC0078:29FC0078.MEAS 0.040810 ...
29NC0007:29NC0007.MEAS 0.003214 ...
32NA1024:32NA1024_SV.MEAS 0.042658 ...
32PC0098:32PE0098.RO03 0.058018 ...
32PI25AJ:32PA25AJ.PNT -0.072946 ...
32PI25AK:32PA25AK.PNT -0.185568 ...
32PICR01:32PACR01.PNT -0.123246 ...
32PICR02:32PACR02.PNT -0.014854 ...
32ZI0350:32ZA0350.PNT 0.079250 ...
38FC0050:38FB0050.RO01 1.000000 ...
40_STEAM:1RVMTR_MW_AI.PNT 0.878075 ...
40HORNS:MW_TEST.RO01 -0.655694 ...
40JI0333:40JI0333.PNT -0.646518 ...
32PC0676:32PE0676.OUT -0.132638 ...
32G558:558TCV1.OUT -0.004901 ...
HMX_WRITE:HR40041.PNT 0.030391 ...
HMX_WRITE:HR40012.PNT 0.023703 ...
32G558:558TCV1.OUT.24hravg -0.011529 ...
LFStorageChestOutletFlow_CALC -0.085409 ...
28FC1030:28FH1030.RI02 0.099777 ...
40PC0063:40PC0063.MEAS 0.198589 ...
PM2_TotalStarch.CV -0.062077 ...
PM2_Rush/Drag.AF -0.072423 ...
32DRVMST_RD:COUCH_CUR.MEAS -0.139073 ...
32DRVMST_RD5:CAL_DRW.MEAS 0.111091 ...
HMX_WRITE:HR40041.PNT HMX_WRITE:HR40012.PNT \
29FC0078:29FC0078.MEAS 0.027260 0.028984
29NC0007:29NC0007.MEAS -0.007062 -0.006434
32NA1024:32NA1024_SV.MEAS -0.055669 -0.050690
32PC0098:32PE0098.RO03 -0.110600 -0.108339
32PI25AJ:32PA25AJ.PNT -0.103705 -0.103143
32PI25AK:32PA25AK.PNT -0.198879 -0.190218
32PICR01:32PACR01.PNT -0.084164 -0.081709
32PICR02:32PACR02.PNT -0.090444 -0.092799
32ZI0350:32ZA0350.PNT 0.049019 0.040712
38FC0050:38FB0050.RO01 0.030391 0.023703
40_STEAM:1RVMTR_MW_AI.PNT 0.002601 -0.001597
40HORNS:MW_TEST.RO01 -0.084078 -0.080841
40JI0333:40JI0333.PNT -0.060688 -0.057895
32PC0676:32PE0676.OUT 0.208618 0.201976
32G558:558TCV1.OUT 0.079870 0.075391
HMX_WRITE:HR40041.PNT 1.000000 0.984959
HMX_WRITE:HR40012.PNT 0.984959 1.000000
32G558:558TCV1.OUT.24hravg 0.065165 0.059095
LFStorageChestOutletFlow_CALC 0.125362 0.126719
28FC1030:28FH1030.RI02 -0.036405 -0.033681
40PC0063:40PC0063.MEAS -0.031109 -0.039507
PM2_TotalStarch.CV 0.024433 0.019269
PM2_Rush/Drag.AF -0.131915 -0.136266
32DRVMST_RD:COUCH_CUR.MEAS -0.005404 -0.005739
32DRVMST_RD5:CAL_DRW.MEAS 0.166845 0.162471
32G558:558TCV1.OUT.24hravg \
29FC0078:29FC0078.MEAS -0.174987
29NC0007:29NC0007.MEAS 0.043894
32NA1024:32NA1024_SV.MEAS -0.166980
32PC0098:32PE0098.RO03 0.282895
32PI25AJ:32PA25AJ.PNT 0.131677
32PI25AK:32PA25AK.PNT -0.193777
32PICR01:32PACR01.PNT -0.027052
32PICR02:32PACR02.PNT 0.106196
32ZI0350:32ZA0350.PNT 0.018780
38FC0050:38FB0050.RO01 -0.011529
40_STEAM:1RVMTR_MW_AI.PNT 0.018824
40HORNS:MW_TEST.RO01 -0.070069
40JI0333:40JI0333.PNT -0.062415
32PC0676:32PE0676.OUT 0.046269
32G558:558TCV1.OUT 0.799331
HMX_WRITE:HR40041.PNT 0.065165
HMX_WRITE:HR40012.PNT 0.059095
32G558:558TCV1.OUT.24hravg 1.000000
LFStorageChestOutletFlow_CALC -0.092040
28FC1030:28FH1030.RI02 0.000808
40PC0063:40PC0063.MEAS -0.001178
PM2_TotalStarch.CV 0.283942
PM2_Rush/Drag.AF 0.319642
32DRVMST_RD:COUCH_CUR.MEAS 0.437746
32DRVMST_RD5:CAL_DRW.MEAS 0.158524
LFStorageChestOutletFlow_CALC \
29FC0078:29FC0078.MEAS 0.020595
29NC0007:29NC0007.MEAS -0.140351
32NA1024:32NA1024_SV.MEAS 0.060482
32PC0098:32PE0098.RO03 -0.491778
32PI25AJ:32PA25AJ.PNT -0.119379
32PI25AK:32PA25AK.PNT -0.061119
32PICR01:32PACR01.PNT 0.183956
32PICR02:32PACR02.PNT -0.186196
32ZI0350:32ZA0350.PNT 0.069739
38FC0050:38FB0050.RO01 -0.085409
40_STEAM:1RVMTR_MW_AI.PNT -0.099897
40HORNS:MW_TEST.RO01 -0.012714
40JI0333:40JI0333.PNT 0.077332
32PC0676:32PE0676.OUT 0.601995
32G558:558TCV1.OUT -0.042119
HMX_WRITE:HR40041.PNT 0.125362
HMX_WRITE:HR40012.PNT 0.126719
32G558:558TCV1.OUT.24hravg -0.092040
LFStorageChestOutletFlow_CALC 1.000000
28FC1030:28FH1030.RI02 -0.027675
40PC0063:40PC0063.MEAS -0.091078
PM2_TotalStarch.CV -0.140004
PM2_Rush/Drag.AF 0.196734
32DRVMST_RD:COUCH_CUR.MEAS -0.087235
32DRVMST_RD5:CAL_DRW.MEAS 0.296776
28FC1030:28FH1030.RI02 40PC0063:40PC0063.MEAS \
29FC0078:29FC0078.MEAS 0.044723 0.046351
29NC0007:29NC0007.MEAS -0.006907 0.066986
32NA1024:32NA1024_SV.MEAS 0.016904 0.054379
32PC0098:32PE0098.RO03 0.374027 0.050872
32PI25AJ:32PA25AJ.PNT 0.013151 0.091954
32PI25AK:32PA25AK.PNT 0.298846 0.082727
32PICR01:32PACR01.PNT -0.083389 -0.031148
32PICR02:32PACR02.PNT 0.078444 0.025130
32ZI0350:32ZA0350.PNT 0.047281 -0.007913
38FC0050:38FB0050.RO01 0.099777 0.198589
40_STEAM:1RVMTR_MW_AI.PNT 0.111939 0.259623
40HORNS:MW_TEST.RO01 0.010090 0.038847
40JI0333:40JI0333.PNT -0.028731 0.023468
32PC0676:32PE0676.OUT -0.336484 -0.109680
32G558:558TCV1.OUT -0.041353 -0.025085
HMX_WRITE:HR40041.PNT -0.036405 -0.031109
HMX_WRITE:HR40012.PNT -0.033681 -0.039507
32G558:558TCV1.OUT.24hravg 0.000808 -0.001178
LFStorageChestOutletFlow_CALC -0.027675 -0.091078
28FC1030:28FH1030.RI02 1.000000 0.071040
40PC0063:40PC0063.MEAS 0.071040 1.000000
PM2_TotalStarch.CV -0.367483 -0.073314
PM2_Rush/Drag.AF -0.137774 -0.079809
32DRVMST_RD:COUCH_CUR.MEAS 0.226462 0.010096
32DRVMST_RD5:CAL_DRW.MEAS -0.320587 -0.049884
PM2_TotalStarch.CV PM2_Rush/Drag.AF \
29FC0078:29FC0078.MEAS -0.239505 -0.129330
29NC0007:29NC0007.MEAS 0.023768 0.051947
32NA1024:32NA1024_SV.MEAS -0.248180 -0.026357
32PC0098:32PE0098.RO03 0.035276 0.279991
32PI25AJ:32PA25AJ.PNT -0.019165 -0.070628
32PI25AK:32PA25AK.PNT -0.405514 -0.316186
32PICR01:32PACR01.PNT -0.021655 -0.011533
32PICR02:32PACR02.PNT 0.007491 -0.039371
32ZI0350:32ZA0350.PNT 0.141518 -0.056810
38FC0050:38FB0050.RO01 -0.062077 -0.072423
40_STEAM:1RVMTR_MW_AI.PNT -0.037104 -0.105279
40HORNS:MW_TEST.RO01 0.005507 -0.063387
40JI0333:40JI0333.PNT 0.017830 -0.017949
32PC0676:32PE0676.OUT 0.100690 0.523273
32G558:558TCV1.OUT 0.303268 0.370837
HMX_WRITE:HR40041.PNT 0.024433 -0.131915
HMX_WRITE:HR40012.PNT 0.019269 -0.136266
32G558:558TCV1.OUT.24hravg 0.283942 0.319642
LFStorageChestOutletFlow_CALC -0.140004 0.196734
28FC1030:28FH1030.RI02 -0.367483 -0.137774
40PC0063:40PC0063.MEAS -0.073314 -0.079809
PM2_TotalStarch.CV 1.000000 0.256495
PM2_Rush/Drag.AF 0.256495 1.000000
32DRVMST_RD:COUCH_CUR.MEAS 0.178520 -0.029527
32DRVMST_RD5:CAL_DRW.MEAS 0.128959 0.399493
32DRVMST_RD:COUCH_CUR.MEAS \
29FC0078:29FC0078.MEAS -0.075676
29NC0007:29NC0007.MEAS -0.019118
32NA1024:32NA1024_SV.MEAS -0.152866
32PC0098:32PE0098.RO03 0.343323
32PI25AJ:32PA25AJ.PNT 0.174564
32PI25AK:32PA25AK.PNT -0.014858
32PICR01:32PACR01.PNT -0.011921
32PICR02:32PACR02.PNT 0.265767
32ZI0350:32ZA0350.PNT 0.093739
38FC0050:38FB0050.RO01 -0.139073
40_STEAM:1RVMTR_MW_AI.PNT -0.091733
40HORNS:MW_TEST.RO01 0.063041
40JI0333:40JI0333.PNT 0.043852
32PC0676:32PE0676.OUT -0.305263
32G558:558TCV1.OUT 0.397024
HMX_WRITE:HR40041.PNT -0.005404
HMX_WRITE:HR40012.PNT -0.005739
32G558:558TCV1.OUT.24hravg 0.437746
LFStorageChestOutletFlow_CALC -0.087235
28FC1030:28FH1030.RI02 0.226462
40PC0063:40PC0063.MEAS 0.010096
PM2_TotalStarch.CV 0.178520
PM2_Rush/Drag.AF -0.029527
32DRVMST_RD:COUCH_CUR.MEAS 1.000000
32DRVMST_RD5:CAL_DRW.MEAS -0.179639
32DRVMST_RD5:CAL_DRW.MEAS
29FC0078:29FC0078.MEAS 0.047845
29NC0007:29NC0007.MEAS 0.018859
32NA1024:32NA1024_SV.MEAS 0.051575
32PC0098:32PE0098.RO03 -0.284449
32PI25AJ:32PA25AJ.PNT -0.145373
32PI25AK:32PA25AK.PNT -0.389314
32PICR01:32PACR01.PNT 0.125589
32PICR02:32PACR02.PNT -0.165420
32ZI0350:32ZA0350.PNT -0.041489
38FC0050:38FB0050.RO01 0.111091
40_STEAM:1RVMTR_MW_AI.PNT 0.080663
40HORNS:MW_TEST.RO01 -0.164282
40JI0333:40JI0333.PNT -0.091574
32PC0676:32PE0676.OUT 0.532187
32G558:558TCV1.OUT 0.168136
HMX_WRITE:HR40041.PNT 0.166845
HMX_WRITE:HR40012.PNT 0.162471
32G558:558TCV1.OUT.24hravg 0.158524
LFStorageChestOutletFlow_CALC 0.296776
28FC1030:28FH1030.RI02 -0.320587
40PC0063:40PC0063.MEAS -0.049884
PM2_TotalStarch.CV 0.128959
PM2_Rush/Drag.AF 0.399493
32DRVMST_RD:COUCH_CUR.MEAS -0.179639
32DRVMST_RD5:CAL_DRW.MEAS 1.000000
[25 rows x 25 columns]
Filled missing values in 29FC0078:29FC0078.MEAS using Random Forest.
Filled missing values in 29NC0007:29NC0007.MEAS using Random Forest.
Filled missing values in 32NA1024:32NA1024_SV.MEAS using Random Forest.
Filled missing values in 32PC0098:32PE0098.RO03 using Random Forest.
Filled missing values in 32PI25AJ:32PA25AJ.PNT using Random Forest.
Filled missing values in 32PI25AK:32PA25AK.PNT using Random Forest.
Filled missing values in 32PICR01:32PACR01.PNT using Random Forest.
Filled missing values in 32ZI0350:32ZA0350.PNT using Random Forest.
Filled missing values in 38FC0050:38FB0050.RO01 using Random Forest.
Filled missing values in 40_STEAM:1RVMTR_MW_AI.PNT using Random Forest.
Filled missing values in 40HORNS:MW_TEST.RO01 using Random Forest.
Filled missing values in 40JI0333:40JI0333.PNT using Random Forest.
Filled missing values in 32G558:558TCV1.OUT using Random Forest.
Filled missing values in HMX_WRITE:HR40041.PNT using Random Forest.
Filled missing values in HMX_WRITE:HR40012.PNT using Random Forest.
Filled missing values in 32G558:558TCV1.OUT.24hravg using Random Forest.
Filled missing values in LFStorageChestOutletFlow_CALC using Random Forest.
Filled missing values in 28FC1030:28FH1030.RI02 using Random Forest.
Filled missing values in 40PC0063:40PC0063.MEAS using Random Forest.
Filled missing values in PM2_TotalStarch.CV using Random Forest.
Filled missing values in PM2_Rush/Drag.AF using Random Forest.
Filled missing values in 32DRVMST_RD5:CAL_DRW.MEAS using Random Forest.
Filled missing values in medium correlation columns using MICE: ['29FC0078:29FC0078.MEAS', '29NC0007:29NC0007.MEAS', '32NA1024:32NA1024_SV.MEAS', '32PC0098:32PE0098.RO03', '32PI25AJ:32PA25AJ.PNT', '32PI25AK:32PA25AK.PNT', '32PICR01:32PACR01.PNT', '32PICR02:32PACR02.PNT', '32ZI0350:32ZA0350.PNT', '38FC0050:38FB0050.RO01', '40_STEAM:1RVMTR_MW_AI.PNT', '40HORNS:MW_TEST.RO01', '40JI0333:40JI0333.PNT', '32PC0676:32PE0676.OUT', '32G558:558TCV1.OUT', 'HMX_WRITE:HR40041.PNT', 'HMX_WRITE:HR40012.PNT', '32G558:558TCV1.OUT.24hravg', 'LFStorageChestOutletFlow_CALC', '28FC1030:28FH1030.RI02', '40PC0063:40PC0063.MEAS', 'PM2_TotalStarch.CV', 'PM2_Rush/Drag.AF', '32DRVMST_RD:COUCH_CUR.MEAS', '32DRVMST_RD5:CAL_DRW.MEAS']
Filled missing values in 29FC0078:29FC0078.MEAS using forward and backward fill.
Filled missing values in 29NC0007:29NC0007.MEAS using forward and backward fill.
Filled missing values in 32NA1024:32NA1024_SV.MEAS using forward and backward fill.
Filled missing values in 32PC0098:32PE0098.RO03 using forward and backward fill.
Filled missing values in 32PI25AJ:32PA25AJ.PNT using forward and backward fill.
Filled missing values in 32PI25AK:32PA25AK.PNT using forward and backward fill.
Filled missing values in 32PICR01:32PACR01.PNT using forward and backward fill.
Filled missing values in 32ZI0350:32ZA0350.PNT using forward and backward fill.
Filled missing values in 38FC0050:38FB0050.RO01 using forward and backward fill.
Filled missing values in 40_STEAM:1RVMTR_MW_AI.PNT using forward and backward fill.
Filled missing values in 40HORNS:MW_TEST.RO01 using forward and backward fill.
Filled missing values in 40JI0333:40JI0333.PNT using forward and backward fill.
Filled missing values in 32G558:558TCV1.OUT using forward and backward fill.
Filled missing values in HMX_WRITE:HR40041.PNT using forward and backward fill.
Filled missing values in HMX_WRITE:HR40012.PNT using forward and backward fill.
Filled missing values in 32G558:558TCV1.OUT.24hravg using forward and backward fill.
Filled missing values in LFStorageChestOutletFlow_CALC using forward and backward fill.
Filled missing values in 28FC1030:28FH1030.RI02 using forward and backward fill.
Filled missing values in 40PC0063:40PC0063.MEAS using forward and backward fill.
Filled missing values in PM2_TotalStarch.CV using forward and backward fill.
Filled missing values in PM2_Rush/Drag.AF using forward and backward fill.
Filled missing values in 32DRVMST_RD5:CAL_DRW.MEAS using forward and backward fill.
/var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:84: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='ffill', inplace=True) /var/folders/yt/3x70v3g11t7bkn4tjh_qdwf40000gn/T/ipykernel_97793/1955705001.py:85: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead. df_clean[col].fillna(method='bfill', inplace=True)
In [155]:
# Check if there are any missing values remaining in the entire DataFrame
missing_data_after_imputation = df_clean.isnull().sum()
# Print the columns with missing values and their counts
missing_columns = missing_data_after_imputation[missing_data_after_imputation > 0]
if missing_columns.empty:
print("No missing data remains after imputation.")
else:
print("Columns with missing data after imputation:")
print(missing_columns)
Columns with missing data after imputation: PM2GrossTPH 1 27FI0023:27FC0023.MEAS 799 27FI0043:27FC0043.MEAS 431 27FI0063:27FC0063.MEAS 28 28FC0011:28FC0011.MEAS 1 29FC0075:29FC0075.MEAS 2 29FC0105:29FC0105.MEAS 43 29FC0114:29FC0114.MEAS 503 29FC0185:29FE0185.RO02 1 29FC0320:29Ft0320.PNT 1 29FC0334:29FC0334.MEAS 2 29FC0334:29FC0334.OUT 1 29FC0351:29FC0351.MEAS 2 29FC0360:29FA0360.PNT 1 29FC1133:29FC1133.MEAS 2 32FC0815:32FB0815.OUT 580 32FC0816:32FQ0816.RO01 1 32FC1013:32FA1013.MEAS 1 32FC1014:32FA1014.MEAS 1 32FI0097:32FA0097.PNT 2987 32FI0097:32FD0097.PNT 1 32FI0264:32FE0264.OUT 1 32FI0643:32FC0643.MEAS 1 32G550A:550FT6.PNT 1 32G550A:550FT7.PNT 1 32HC0055:32HB0055.OUT 19 39FC0071:39FA0071.MEAS 2 40FC0387:40FC0387.MEAS 1 44FT0011:44FT0011.PNT 276 ASA_SIZE:FLOW_PM2.PNT 1 PM2.TotalSweetnerFlow 1513 32FC0620:32FD0620.OUT 1 29FC0074:29FC0074.OUT 18 29FC0352:29FC0352.MEAS 1 29FI0018:29FI0018.PNT 4 29FC0079:29FC0079.MEAS 128 29LC0113:29LC0113B.MEAS 3 32FC0669:32FD0669.OUT 1 32HC0060:32HA0060.PNT 1 32FC0677:32FD0677.OUT 1 ASA_SIZE:CALC_PM2.RO02 8 32FC0425:32FA0425.PNT 1 27FI0411:27FA0411.PNT 6 28FC1141:28FG1141.PNT 5 28FC0732:28FC0732.MEAS 5 18FC0801:18FC0801.MEAS 3 18LC0020:18FD0020.RO0001 6 19FC0206:19FG0206.RO04 1 Crse_Scrn.Total 4 12FI0110:12FI0110.PNT 56 dtype: int64
In [166]:
# Check and handle columns with missing values
missing_columns = missing_data_after_imputation[missing_data_after_imputation > 0]
for col in missing_columns.index:
if pd.api.types.is_numeric_dtype(df_clean[col]):
# If the column is numeric, fill missing values with the mean
df_clean[col].fillna(df_clean[col].mean(), inplace=True)
else:
# If the column is not numeric, choose another strategy, such as filling with the mode
df_clean[col].fillna(df_clean[col].mode()[0], inplace=True)
# Check for missing values after filling
missing_data_after_imputation = df_clean.isnull().sum()
missing_columns = missing_data_after_imputation[missing_data_after_imputation > 0]
if missing_columns.empty:
print("No missing data remains after imputation.")
else:
print("Columns with missing data after imputation:")
print(missing_columns)
No missing data remains after imputation.
In [167]:
# Sort the data by the number of columns in descending order
sorted_column_counts = dict(sorted(column_counts.items(), key=lambda item: item[1], reverse=True))
# Plot the sorted bar chart
plt.figure(figsize=(12, 6))
plt.barh(list(sorted_column_counts.keys()), list(sorted_column_counts.values()), color='skyblue')
plt.xlabel('Number of Columns')
plt.ylabel('Sensor Categories')
plt.title('Number of Columns per Sensor Category')
plt.tight_layout()
plt.show()
In [168]:
other_columns = column_classification['Other Sensors']
print(other_columns)
['29FC0078:29FC0078.MEAS', '29NC0007:29NC0007.MEAS', '32NA1024:32NA1024_SV.MEAS', '32PC0098:32PE0098.RO03', '32PI25AJ:32PA25AJ.PNT', '32PI25AK:32PA25AK.PNT', '32PICR01:32PACR01.PNT', '32PICR02:32PACR02.PNT', '32ZI0350:32ZA0350.PNT', '38FC0050:38FB0050.RO01', '40_STEAM:1RVMTR_MW_AI.PNT', '40HORNS:MW_TEST.RO01', '40JI0333:40JI0333.PNT', '32PC0676:32PE0676.OUT', '32G558:558TCV1.OUT', 'HMX_WRITE:HR40041.PNT', 'HMX_WRITE:HR40012.PNT', '32G558:558TCV1.OUT.24hravg', 'LFStorageChestOutletFlow_CALC', '28FC1030:28FH1030.RI02', '40PC0063:40PC0063.MEAS', 'PM2_TotalStarch.CV', 'PM2_Rush/Drag.AF', '32DRVMST_RD:COUCH_CUR.MEAS', '32DRVMST_RD5:CAL_DRW.MEAS']
In [174]:
df_clean
Out[174]:
| TimeStamp | QualityMeasure | 32PRODGRADE | PM2GrossTPH | 18LI0028:18LA0028.PNT | 19FC0002:19FC0002.MEAS | 19LI0472:19LI0472.PNT | 19LI0477:19LA0477.PNT | 22AUTOMAX_RD:1STDRYER_DFB.PNT | 22AUTOMAX_RD:PERCENT_DRAW.RO01 | ... | 32DRVMST_RD4:5THDRYER_CUR.MEAS | 32DRVMST_RD4:5THDRYER_DRW.MEAS | 32DRVMST_RD4:6THDRYER_CUR.MEAS | 32DRVMST_RD4:6THDRYER_DRW.MEAS | 32DRVMST_RD5:7THDRYER_CUR.MEAS | 32DRVMST_RD5:7THDRYER_DRW.MEAS | 32DRVMST_RD5:CAL_DRW.MEAS | 32DRVMST_RD6:REELDRUM_DRW.MEAS | 32DRVMST_RD6:REELDRUM_CUR.MEAS | 12FI0110:12FI0110.PNT | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2022-07-01 00:30:06 | 51.500000 | ME23 | 61.333 | 69.304850 | 0.000701 | 88.918460 | 39.331660 | 60.343870 | 1.857722 | ... | 30.911760 | 3.038673 | 22.560300 | 3.077353 | 59.680890 | 3.521016 | 3.712504 | -7.833509 | 32.779830 | 5478.449962 |
| 1 | 2022-07-01 02:27:09 | 51.500000 | ME23 | 61.22888 | 67.751520 | 0.000701 | 87.771370 | 39.168230 | 59.702350 | 1.850324 | ... | 30.000000 | 2.099999 | 20.231380 | 3.370393 | 59.380340 | 3.133648 | 3.200000 | -6.768645 | 32.535390 | 5383.978364 |
| 2 | 2022-07-01 03:35:05 | 51.666668 | ME23 | 60.02072 | 58.737290 | 0.000701 | 88.662290 | 39.073380 | 58.697950 | 1.826701 | ... | 31.000000 | 1.951585 | 20.000000 | 3.131916 | 57.001580 | 3.874452 | 3.577793 | -5.394183 | 32.499210 | 5445.179291 |
| 3 | 2022-07-02 04:26:49 | 50.916668 | ME23 | 59.77631 | 70.728870 | 0.000701 | 84.096380 | 37.361330 | 54.172070 | 1.647153 | ... | 30.000000 | 3.000644 | 20.716260 | 2.618661 | 62.859860 | 3.227230 | 3.200000 | -5.394183 | 32.513280 | 5740.561517 |
| 4 | 2022-07-02 05:01:01 | 51.333332 | ME23 | 59.82626 | 66.938570 | 0.000701 | 85.309300 | 37.310670 | 54.432180 | 1.657713 | ... | 29.734050 | 3.412886 | 19.632880 | 2.900000 | 59.467300 | 3.383340 | 3.483357 | -9.834175 | 31.844340 | 5496.862867 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 3264 | 2024-04-20 23:12:08 | 54.500000 | ME23 | 60.143318 | 61.280479 | 0.089437 | 87.388474 | 54.804222 | 66.018845 | 2.101269 | ... | 21.454918 | 6.653681 | 21.315865 | 0.096677 | 33.701187 | 2.113836 | 15.068970 | -14.211590 | 39.717907 | 3938.786707 |
| 3265 | 2024-04-21 00:20:35 | 54.000000 | ME23 | 59.95488 | 50.145210 | 0.038160 | 91.183426 | 54.751965 | 66.292595 | 2.109345 | ... | 21.676327 | 3.387486 | 21.179834 | 0.676058 | 34.193932 | 2.696882 | 13.289554 | -15.003655 | 39.018269 | 3856.163950 |
| 3266 | 2024-04-21 01:31:49 | 53.333332 | ME23 | 56.970501 | 52.800426 | 0.101979 | 86.527016 | 54.694817 | 57.609299 | 1.907183 | ... | 20.083513 | 4.565936 | 21.827986 | 1.656811 | 30.285254 | 1.145926 | 13.768075 | -15.192116 | 39.079048 | 3759.583816 |
| 3267 | 2024-04-23 08:44:20 | 55.166668 | ME23 | 42.047024 | 59.331093 | 48.501858 | 87.588478 | 50.500904 | 60.305599 | 1.962264 | ... | 17.503027 | 3.006409 | 18.554209 | 1.585764 | 24.350523 | 0.557838 | 2.824477 | -4.000000 | 44.418255 | 3733.627990 |
| 3268 | 2024-04-23 09:20:41 | 55.166668 | ME23 | 42.062473 | 68.735703 | 49.230766 | 90.254387 | 50.409645 | 60.232826 | 1.962190 | ... | 17.305225 | 3.123813 | 18.688637 | 1.159925 | 23.925316 | 0.903451 | 2.641837 | -4.000000 | 44.630287 | 3825.232948 |
3269 rows × 522 columns
In [178]:
output_csv_path = 'cleaned_dataset.csv'
df_clean.to_csv(output_csv_path, index=False, encoding='utf-8-sig')
In [180]:
# Create an empty dictionary to store the classification for each sensor ID
sensor_classification = {}
# Map each sensor ID to its group by iterating through column_classification
for category, sensor_ids in column_classification.items():
for sensor_id in sensor_ids:
sensor_classification[sensor_id] = category
# Create a new column 'Category' and insert it into the sensor_data DataFrame
sensor_data['Category'] = sensor_data['Sensor ID'].map(sensor_classification)
# Display the DataFrame with the newly added column
print(sensor_data.head())
Sensor ID Description \
0 18LI0028:18LA0028.PNT De-Ink Stock Chest Level
1 19FC0002:19FC0002.MEAS DI2 Stock To #1 Blend
2 19LI0472:19LI0472.PNT Luredur Finished Product Tank Level
3 19LI0477:19LA0477.PNT Center Roll Cleaner Tank Level
4 22AUTOMAX_RD:1STDRYER_DFB.PNT 1st Dryer-Center Roll Draw
Category
0 Tank Level
1 Blend
2 Tank Level
3 Tank Level
4 Dryer
In [182]:
output_csv_path = 'SensorData_Description.csv'
sensor_data.to_csv(output_csv_path, index=False, encoding='utf-8-sig')
In [ ]: